diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a99d75b --- /dev/null +++ b/.gitignore @@ -0,0 +1,452 @@ + +# Created by https://www.gitignore.io/api/c,macos,linux,latex,python,windows + +### Custom ### +# remove test file iters.db +*.db + +### C ### +# Prerequisites +*.d + +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + +### LaTeX ### +## Core latex/pdflatex auxiliary files: +*.aux +*.lof +*.log +*.lot +*.fls +*.toc +*.fmt +*.fot +*.cb +*.cb2 + +## Intermediate documents: +*.dvi +*.xdv +*-converted-to.* +# these rules might exclude image files for figures etc. +# *.ps +# *.eps +# *.pdf + +## Generated if empty string is given at "Please type another file name for output:" +.pdf + +## Bibliography auxiliary files (bibtex/biblatex/biber): +*.bbl +*.bcf +*.blg +*-blx.aux +*-blx.bib +*.run.xml + +## Build tool auxiliary files: +*.fdb_latexmk +*.synctex +*.synctex(busy) +*.synctex.gz +*.synctex.gz(busy) +*.pdfsync +*Notes.bib + +## Auxiliary and intermediate files from other packages: +# algorithms +*.alg +*.loa + +# achemso +acs-*.bib + +# amsthm +*.thm + +# beamer +*.nav +*.pre +*.snm +*.vrb + +# changes +*.soc + +# cprotect +*.cpt + +# elsarticle (documentclass of Elsevier journals) +*.spl + +# endnotes +*.ent + +# fixme +*.lox + +# feynmf/feynmp +*.mf +*.mp +*.t[1-9] +*.t[1-9][0-9] +*.tfm + +#(r)(e)ledmac/(r)(e)ledpar +*.end +*.?end +*.[1-9] +*.[1-9][0-9] +*.[1-9][0-9][0-9] +*.[1-9]R +*.[1-9][0-9]R +*.[1-9][0-9][0-9]R +*.eledsec[1-9] +*.eledsec[1-9]R +*.eledsec[1-9][0-9] +*.eledsec[1-9][0-9]R +*.eledsec[1-9][0-9][0-9] +*.eledsec[1-9][0-9][0-9]R + +# glossaries +*.acn +*.acr +*.glg +*.glo +*.gls +*.glsdefs + +# gnuplottex +*-gnuplottex-* + +# gregoriotex +*.gaux +*.gtex + +# hyperref +*.brf + +# knitr +*-concordance.tex +# TODO Comment the next line if you want to keep your tikz graphics files +*.tikz +*-tikzDictionary + +# listings +*.lol + +# makeidx +*.idx +*.ilg +*.ind +*.ist + +# minitoc +*.maf +*.mlf +*.mlt +*.mtc[0-9]* +*.slf[0-9]* +*.slt[0-9]* +*.stc[0-9]* + +# minted +_minted* +*.pyg + +# morewrites +*.mw + +# nomencl +*.nlo + +# pax +*.pax + +# pdfpcnotes +*.pdfpc + +# sagetex +*.sagetex.sage +*.sagetex.py +*.sagetex.scmd + +# scrwfile +*.wrt + +# sympy +*.sout +*.sympy +sympy-plots-for-*.tex/ + +# pdfcomment +*.upa +*.upb + +# pythontex +*.pytxcode +pythontex-files-*/ + +# thmtools +*.loe + +# TikZ & PGF +*.dpth +*.md5 +*.auxlock + +# todonotes +*.tdo + +# easy-todo +*.lod + +# xindy +*.xdy + +# xypic precompiled matrices +*.xyc + +# endfloat +*.ttt +*.fff + +# Latexian +TSWLatexianTemp* + +## Editors: +# WinEdt +*.bak +*.sav + +# Texpad +.texpadtmp + +# Kile +*.backup + +# KBibTeX +*~[0-9]* + +# auto folder when using emacs and auctex +/auto/* + +# expex forward references with \gathertags +*-tags.tex + +### LaTeX Patch ### +# glossaries +*.glstex + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + + +# End of https://www.gitignore.io/api/c,macos,linux,latex,python,windows diff --git a/AUTHORS b/AUTHORS new file mode 100755 index 0000000..d79a473 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,4 @@ +Authors of FLAViz: + +* Krishna Raj Devkota (ETACE, Bielefeld University) +* Sander van der Hoog (ETACE, Bielefeld University) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..70566f2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ +GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. \ No newline at end of file diff --git a/README.md b/README.md index 5136a4f..17717ae 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,137 @@ -# Data processing and analyzing scripts + FLAViz: Flexible Large-scale Agent Visualization Library -#### Depends +------------------------------------------------------------------------------- -Python core libraries -Pandas -matplotlib -h5py -pytables -scipi -numy + Copyright (c) 2018 Sander van der Hoog + If you have any problems or enquiries, you can contact the + library maintainer at -#### Usage +------------------------------------------------------------------------------- -Refer to individual folders +The Flexible Large-scale Agent Visualization Library (FLAViz) is a data analysis and agent visualization library developed for agent-based simulation models created using the FLAME simulation environment. + +Agent-based simulations typically generate data spanning multiple dimensions, e.g. parameter sets, Monte Carlo replication runs, different agent types, many agent instances per type, many variables per agent, and the time dimension. + +To deal with such a large heterogeneity and variety of data, it should be stored as an organized, high-dimensional data set, to allow for proper data aggregation, filtering, selection, slicing etc. + +FLAViz builds on Python pandas and matplotlib, and can be used to filter, transform, and visualize time series data that is stored using hierarchical levels in the HDF5 file format. Various plotting styles can be specified, e.g., time series, box plots, scatter plots, histograms, and delay- or phase plots. + +------------------------------------------------------------------------------- +Features +-- +v1.0 - First release + + * data conversion from XML to SQLite DB and HDF5 + * data module using Python pandas + * visualization module based on matplotlib + + +Getting started +-- + +For more information on how to use the library, view the [manual](https://github.com/svdhoog/FLAViz/blob/master/docs/manual/tex/FLAViz_Manual.pdf). + +Many example plots are [here](https://github.com/svdhoog/FLAViz/tree/master/docs/visualisation_scripts/readme.rst). + +To get started immediately, you can run the library out-of-the-box using the [test data](https://github.com/svdhoog/FLAViz/tree/master/data/visualisation). Instructions on how to do this can be found in the section on Testing in the [tutorial](https://github.com/svdhoog/FLAViz/tree/master/docs/visualisation_scripts/readme.rst). + +Test data +-- + +A simple model with test data is provided in the folder [circles model](https://github.com/svdhoog/FLAViz/tree/master/data/visualisation/models/circle). A description is in the [readme file](https://github.com/svdhoog/FLAViz/tree/master/data/visualisation/models/circle/readme.rst). + + +Requirements: +-- + +You will need to satisfy the following dependencies. + +For the FLAViz visualization scripts: + +* python3 (>= 3.4) Python3 core libraries +* python3-h5py (>= 2.2.1) -- needed for HDF5 files to be re-writable, appendable +* python3-matplotlib (>= 2.2.2) +* python3-numexp (>= 2.6) +* python3-numpy (>= 1.14.3) +* python3-pandas (0.22.0 or >= 0.19.0) +* python3-pyTables (>= 3.4) -- currently used for HDF5 files, using the 'tables' format (slower than 'fixed' format, but allows for appendable HDFStores) +* python3-scipy (1.1.0) +* python3-tk (>= 3.4.3) +* python3-yaml (>= 3.12) +* python3-lxml (4.2.1 or >= 3.3.3) + +For the data conversion scripts: + +* sqlite3 (>= 3.8.2) +* python3-pandas (0.22.0 or >= 0.19.0) +* python3-lxml (4.2.1 or >= 3.3.3) + +Installation: +-- + +To see which versions of the Python3 packages you have installed, run: + +``` +$ ls /usr/local/lib/python3.4/dist-packages -lh +``` + +Notes: +-- +On some platforms the package manager will install older versions of the above packages. +For example, in Linux Mint 17 (Quiana), the version of matplotlib that is installed using apt-get is 1.19 instead of the newer version 2.2.2 (which is required). + +The following packages can be installed via your distribution's package manager: + +``` +$ sudo apt-get install sqlite3 python3-pip python3-lxml python3-yaml python3-numexp python3-tables python3-tk +``` + +To install the latest versions of the Python numerical packages, you can use the python3-pip command: + +``` +$ python3 -mpip install -U matplotlib numpy pandas scipy +``` + +Alternatively, you can us pip3 directly: + +``` +$ pip3 install matplotlib numpy pandas scipy +``` + +Running the main module +-- + +The main Python script has a single argument, which is the path to the folder containing configuration files: +To start the library, you should run it from the root FLAViz folder: + +``` +$ python3 /src/visualisation_scripts/main.py -p /src/visualisation_scripts/config +``` +where `config` is the path to the config folder containing the configuration files: + +* config.yaml +* plot_config.yaml +* config_transform.yaml [optional] + +For more information please see the `docs/` folder. + + +Contributors +------------ + - [Sander van der Hoog](https://github.com/svdhoog) + - [Krishna Raj Devkota](https://github.com/krdevkota) + - [Fabian Herrmann](https://github.com/0xfabi) + +Contributing +------------ + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`) +3. Commit your changes (`git commit -am 'Added some feature'`) +4. Push to the branch (`git push origin my-new-feature`) +5. Create new Pull Request + +Copyright +--------- + +Copyright (c) 2018 Sander van der Hoog. See LICENSE for further details. diff --git a/TODO/alternate.py b/TODO/alternate.py deleted file mode 100644 index 57a34b5..0000000 --- a/TODO/alternate.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -import sys, os, argparse -import numpy as np -import pandas as pd - -def process_hdf_keys( string_in ): - def find_between( s, first, last ): - try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) - return s[start:end] - except ValueError: - return "" - - tmp_string = string_in.replace('_run_', ',') - string_out = find_between(tmp_string,"/set_","_iters") - return list(map(int, string_out.split(','))) - -# Start with the main part - -store = pd.io.pytables.HDFStore('/home/susupta/Desktop/GitHub/Bank/Bank.h5') -d = pd.DataFrame() -df_list =[] -for key in store.keys(): - sets_runs = process_hdf_keys(key) - s = sets_runs[0] - r = sets_runs[1] - # Opening Panel the particular set and run - pnl = store.select(key) - - # Converting panel to Dataframe - df = pnl.to_frame() - - # Adding two columns for set and run into the dataframe for two added level of indexing - df['set'] = s - df['run'] = r - df.set_index('run', append = True, inplace = True) - df.set_index('set', append = True, inplace = True) - df_list.append(df.reorder_levels(['set', 'run', 'major', 'minor'])) - del df -# make sure df on the list have same columns -d = pd.concat(df_list) -del df_list -# If columns of items in df_list are not same use this instead (not tested yet): -#d = pd.DataFrame.from_dict(map(dict,df_list)) - -store.close() diff --git a/TODO/doc/dataset.rst b/TODO/doc/dataset.rst deleted file mode 100644 index 353b973..0000000 --- a/TODO/doc/dataset.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _dataset: - -Input and Output data storage format -==================================== - -The input data is stored in a ``HDF5`` container file (*.h5, .hdf5*) in a hierarchial form. - -Currently, the ``HDF5`` file is arranged as follows: - -- Each agent-type is contained in a separate HDF5 file, with the same name. - -- Each HDF5 file has a single hierarchy, with the ``agent-type`` as the root, and the ``set`` and ``runs`` as the branches. - -- the set and runs branches each contain a ``Pandas 3D data Panel``, which contains the ``major``, ``minor``, and ``items`` axis. - -- the ``Pandas 3D data Panel`` is written to the HDF5 file with the help of ``Pytable`` module of python. - - -A HDF5 file as described above can be created from the SQLite db files by using the data processing scripts, included in the data processing directory - -*Note:* To avoid any unwanted errors, it is imperative to name the ``SQLite`` db files with the following convention: ``set_*_run_*_iters.db`` - - -**References:** - - Python Pandas: http://pandas.pydata.org/ - - HDF5: https://support.hdfgroup.org/HDF5/ - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/TODO/doc/parameters.rst b/TODO/doc/parameters.rst deleted file mode 100644 index fca68ec..0000000 --- a/TODO/doc/parameters.rst +++ /dev/null @@ -1,309 +0,0 @@ -There are three parameter files, through which the necessary conditions can be input. The parameter files have a -hierarchial format, not just for clarity but also for functionality. Hence, it is very important to abide by the -indentation for the input to be interpreted correctly. - -*Note:* Any error in the yaml file might be caught by the exception handler, but indentation errors go unnoticed -sometimes, therefore resulting in undesired output. Hence, extra care is advised when formulating a parameter file. - -main.yaml -~~~~~~~~~ - -``i/o``: Specify the input and output path, in the subhierarchies ``input_path`` and ``output_path``:: - - i/o: - input_path: '/home/etace/Desktop/GitHub/HDFDataFiles' - - input_files: - CentralBank: CentralBank.h5 - Eurostat: Eurostat.h5 - Firm: Firm.h5 - -*Note:* The key name to the input files should correspond to the Agent-type (i.e. Bank, Eurostat, Firm etc.) - -Similarly, the output path:: - - output_path: '/home/etace/Spaces/ETACE-Conquaire Project/Data/data_reproduction_task/timeseries_classify_LS_output' - - -**Plot-key (i.e.** ``plot1`` **) :** Specify a key for the plot (mainly to keep track of the plot-number for other configuration files). Can be any string. - - -**Plot-type (i.e.** ``timeseries`` **) :** Nested under **Plot-key (** ``plot1`` **)**, "Plot-type" specifies the type of plot desired. *Possible types:* ``timeseries``, ``boxplot``, ``histogram``, ``scatterplot`` - - -*Note [Exception]*: For the case of transform, simply specify ``transform`` in the Plot-type, and it will perform the transform (no plots will be produced). - - - -``agent``: Name of the agent-type, nested under **Plot-type**. - -``analysis``: Type of analysis. *Possible types:* ``agent``, ``multiple_run``, ``multiple_batch``, ``multiple_set``. - -``variables``: Variables from the particular agent-type that is to be processed/ visualized. The sub-hierarchy ``var1``, ``var2`` etc. allows -the input of multiple variables for any agent type. The variable names can be inside a set of square braces *[]* or simply inside a set of single-quotation marks *''*. - - -Example:: - - plot1: - timeseries: - agent: Bank - analysis: multiple_set - variables: - var1: [total_credit] - var2: [equity] - - -Example (*only for Transform*):: - - plot1: - transform: - agent: Bank - analysis: multiple_set - variables: - var1: [total_credit] - var2: [equity] - - - -``conditional_filtering`` : There is an option of filtering the variables that meet certain criteria i.e. -get only those values of the variables that satisfy a certain upper/lower limit or fall within a certain range. - -For conditional filtering, specify the variables as above, but with the filter conditions in place. *Possible operator types:* ``<``, ``>``, ``<=``, ``>=``, ``==``. - -Example:: - - var1: [variable name, 'operator[value]'] - e.g. - var1: [total_credit, '>[700]'] # selects those values of total credit variable that are greater than 700 - - - var2: [variable name, 'operator1[value]','operator2[value]'] - e.g. - var2: [equity, '>[700]', '<[1500]'] # selects those values of total credit variable that are between 700 and 1500 - - -Usage Example:: - - plot1: - timeseries: - agent: Bank - analysis: multiple_set - variables: - var1: [total_credit, '>[700]'] - var2: [equity, '>[700]', '<[800]'] - - - - -Similarly, for desired **sets**, **runs**, **major** and **minor** axes, filter conditions can be specified as necessary. - -``set`` : Specify the set numbers to be processed. Input can be a list ( *Case I* ), or ( *esp. for long lists*) a custom way -to specify the range of values ( *Case II* ). - -Example:: - - Case I: - set: [val(1),val(2),...,val(N)] - e.g. - set: [1,2] # list with values 1 and 2 - - Case II: - set: [range,[val(1),val(N),step-size]] - e.g. - set: [range, [1,10,2]] # list with values from 1 to 10 with a step-size of 2 - - -``run`` : Specify the runs to process. Syntax similar to ``set`` above. - -``major`` : Specify the values from the major axis (time periods) to process. Syntax similar to ``set`` above. - -``minor`` : Specify the values from the minor axis (agent instances) to process. Syntax similar to ``set`` above. - -*Note:* The ``set``, ``run``, ``major``, and ``minor`` values are nested under "Plot-type" - -Example:: - - plot1: - timeseries: - agent: Bank - analysis: multiple_set - variables: - var1: [total_credit] - var2: [equity] - set: [1] - run: [1,2] - major: [range,[6020,26000,20]] - minor: [1,5,7] - - -``summary``: Specify the type of statistical summary. This is also nested under Plot-type. - -Possible types : ``mean``, ``median``, ``custom_quantile``, ``upper_quartile``, ``lower_quartile``, ``maximum``, ``minimum``. - - -Example:: - - plot1: - timeseries: - summary: mean - - -Hence, a typical main configuration file may look like:: - - i/o: - input_path: - Bank: '/home/etace/Desktop/GitHub/Bank/Bank.h5' # please name the key as the agent name - Eurostat: '/home/etace/Desktop/GitHub/Bank/Eurostat.h5' - output_path: '/home/etace/Spaces/ETACE-Conquaire Project/Data/data_reproduction_task/timeseries_classify_LS_output' - - plot1: - timeseries: - agent: Bank - analysis: multiple_set - variables: - var1: [total_credit] - var2: [equity] - set: [1] - run: [1,2] - major: [range,[6020,26000,20]] - minor: [1,5,7] - summary: mean - - plot2: - boxplot: - agent: Eurostat - analysis: multiple_run - variables: - var1: [total_credit] - set: [1] - run: [1] - major: [range,[6020,6900,20]] - minor: [1,8] - summary: custom_quantile - - - -plot.yaml -~~~~~~~~~ - -The *plot.yaml* file contains all the necessary configurations for a plot that is output. Whenever a plot is specified on the *main.yaml* file, the plot.yaml file is read for the necessary -specifications of the plot. -As such, some of the parameters from the *plot.yaml* file is explained below: - -**Plot-key( i.e.** ``plot1`` **):** This string should be the same as the Plot-key in the main.yaml file, to make sure -the correct parameters are mapped to the respective plot. - -``number_plots``: Specifies how many plots will be output per variable for a particular agent type. -*Possible values:* ``one``, ``many``. - -``plot_name``: Specify filename for the plot. - -*Note:* In case of multiple plots, a numerical suffix (*in increasing order*) is added after the specified file name. - -``plot_legend``: Specify legend for the plot. - -``legend_loc``: Specify location of the legend, either inside the box or outside of it. *Possible values:* ``in``, ``out``. - -``legend_label``: Specify name for the lines in the plot. Can be any string value. - -``x-axis label``: Specify label for the x-axis. Can be any combination of string values. - -``y-axis label``: Specify label for the y-axis. Can be any combination of string values. - -``linestyle``: Specify line characteristic. *Possible values:* ``solid``, ``dashed``, ``dashdot``, ``dotted`` etc. - - -Therefore, a particular *plot.yaml* file might look like:: - - plot1: - number_plots: one - plot_name: timeseries_equity.png - l_lim: no - u_lim: no - tmin: no - tmax: no - plot_legend: yes - legend_loc: out - legend_label: equity - x-axis label: months - y-axis label: equity value - linestyle: solid - - - plot2: - number_plots: many - plot_name: boxplot_monthly_output.png - l_lim: no - u_lim: no - tmin: no - tmax: no - plot_legend: no - legend_loc: in - legend_label: monthly_output - x-axis label: months - y-axis label: monthly_output - linestyle: dashed - - -transform.yaml -~~~~~~~~~~~~~~ - -The *transform.yaml* file contains all the necessary configurations for any transformation specified on the *main.yaml* file. Whenever a transformation is specified on the *main.yaml* file, the *transform.yaml* file is read for the necessary -specifications of the plot. - -As such, some of the parameters from the *transform.yaml* file is explained below: - -**Plot-key( i.e.** ``plot1`` **):** This string should be the same as the Plot-key in the main.yaml file, to make sure -the correct parameters are mapped to the respective plot. - -*Note:* Although it is called Plot-key, the transform case is an exception and no plots are produced in transform case. - -``variables``: Variables from the particular agent-type that is to be transformed. The sub-hierarchy ``var1``, ``var2`` etc. allows -the input of multiple variables for any agent type. - -``transform_function``: The transformation function to apply for the given variables to produce the necessary transforms. - -*Possible functions:* - -- Quarterly growth rate (quarter on quarter freq quaterly) ``q_o_q_q`` -- Quarterly growth rate (quarter on quarter freq annual) ``q_o_q_a`` -- Monthly growth rate (month on month freq annual) ``m_o_m_a`` -- Monthly growth rate (month on month freq monthly) ``m_o_m_m`` -- Annual growth (year on year freq annual) ``y_o_y_a`` -- Other custom functions - -*Note:* Other elementary functions such as **sum**, **difference**, **product**, and **division** can also be performed, which has been left for the user (will be added as custom functions). - -``aggregate``: If the transformation is to be performed after calculating the summary stats, then a necessary aggregation method can be specified. -*Possible values:* ``mean``, ``median``, ``maximum``, ``minimum``, ``custom_quantile``, ``upper_quartile``, ``lower_quartile``. - -``write_file``: Specify whether to write the transformation as a file. *Possible values:* ``yes``, ``no``. - -``output_path``: If the ``write_file`` option above is set to ``yes``, then a output path for the file needs to be specified. -Can be any valid filepath, as a string, including upto the filename. - -``hdf_groupname``: Specify the rootname for the HDF5 group name (internal hierarchy) for the transformed variable. Can be any valid string. - -A particular *transform.yaml* file may, therefore, look as follows:: - - plot2: - variables: - var1: total_credit - var2: equity - transform_function: m_o_m_ONE_CYCLE - aggregate: mean - - write_file: yes - output_path: '/home/etaceguest/transform_test/transform_out/transformed.h5' - hdf_groupname: 'total_credit_equity_ratio' - - - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**References:** - - Matplotlib: https://matplotlib.org/ - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/TODO/doc/tutorial.rst b/TODO/doc/tutorial.rst deleted file mode 100644 index 408917f..0000000 --- a/TODO/doc/tutorial.rst +++ /dev/null @@ -1,312 +0,0 @@ -.. _tutorial: - -Possible Uses -============= - -The module can be used to filter, transform, and visualize time series data, in multiple hierarchial levels. - -The different kinds of plots that are possible are: time series, box plot, scatter plot, histogram, and delay plot. - -Structure -========= - -There are several scripts, located in folder ``/src`` that deal with different steps during the data visualisation/transformation processes: - -- ``main.py`` : Contains code to read in the input data and primary parameter files, filter the data based on necessary conditions, and link the different Summary, Plot, and Transformation modules. -- ``summarystats.py`` : Takes in a Pandas dataframe, and computes the necessary summary as needed, and outputs the result as a Pandas dataframe. -- ``plots.py``: Takes in a Pandas dataframe, and returns the necessary plots as specified. -- ``transform.py`` : Takes in a Pandas dataframe, and returns/ writes to a file the necessary transformations. - -The parameter files are contained in the folder named ``/config``. It contains three configuration files: - -- ``main.yaml`` : define i/o path, plot-types, agents and appropriate variables, filter options, desired summary -- ``plot.yaml`` : define plot properties i.e. name, legends, line style, fill etc. -- ``transform.yaml``: define variables to transform, type of data transformations, i/o info to store data in a file after transformations - - -Note: Further details on how to use the parameter (yaml) files can be found on the ``/parameters.rst`` file of the documentation. - -Testing -======= - -To run the module, simply use: - - $ python main.py parameter-path - -where, parameter path is the path to the folder containing the yaml files - -Note: The module has only been tested under Unix/ Unix-like systems. It has not been tested for Windows and therefore -there is no guarentee of proper execution for such systems. - -Examples -======== - -Once the necessary parameters are set, by following the instructions specified in *parameters.rst* file, the module can be run to get the desired results. To demonstrate some of the functionalities, -the parameters of the configuration files are shown below, along with the plots they yield. - - -Using the dataset , and the following parameter settings, the following plots can be produced. - -**Example 1** (*For agent Firm, four sets, twenty runs each, eighty instances, plotted in a single plot* ): - -main.yaml:: - - plot1: - timeseries: - agent: Firm - analysis: multiple_batch - variables: - var1: [technology] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [range,[1,80]] - summary: mean - -plot.yaml:: - - plot1: - number_plots: one - plot_name: timeseries_agentanalysis.png - plot_legend: yes - legend_location: best - x-axis label: technology - y-axis label: Time - linestyle: dashed - marker: None - - -.. image:: ./images/P1_ts_technology.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right - - -**Example 2** (*For agent Firm, four sets, twenty runs each, eighty instances, plotted in a single plot* ): - -main.yaml:: - - plot2: - timeseries: - agent: Firm - analysis: multiple_batch - variables: - var1: [total_debt] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [range,[1,80]] - summary: mean - -plot.yaml:: - - plot2: - number_plots: one - plot_name: timeseries_agentanalysis.png - plot_legend: yes - legend_location: best - x-axis label: Time - y-axis label: total_debt - linestyle: dashed - marker: None - - -.. image:: ./images/P2_ts_total_debt.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right - - -**Example 3** (*For agent Firm, four sets, twenty runs each, eighty instances, plotted in a single plot* ): - -main.yaml:: - - plot3: - timeseries: - agent: Firm - analysis: multiple_batch - variables: - var1: [output] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [range,[1,80]] - summary: mean - -plot.yaml:: - - plot3: - number_plots: one - plot_name: timeseries_agentanalysis.png - plot_legend: yes - legend_location: best - x-axis label: Time - y-axis label: output - linestyle: dashed - marker: None - - -.. image:: ./images/P3_ts_output.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right - - -**Example 4** (*For agent Firm, four sets, twenty runs each, eighty instances, plotted in a single plot* ): - -main.yaml:: - - plot4: - timeseries: - agent: Firm - analysis: multiple_batch - variables: - var1: [price] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [range,[1,80]] - summary: mean - -plot.yaml:: - - plot3: - number_plots: one - plot_name: timeseries_agentanalysis.png - plot_legend: yes - legend_location: best - x-axis label: Time - y-axis label: price - linestyle: dashed - marker: None - - -.. image:: ./images/P4_ts_price.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right - - - -**Example 5** (*For agent Eurostat, four sets, twenty runs each, one instance each, plotted in a single plot* ): - -main.yaml:: - - plot5: - timeseries: - agent: Eurostat - analysis: multiple_batch - variables: - var1: [unemployment_rate] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [0] - summary: mean - -plot.yaml:: - - plot5: - number_plots: one - plot_name: timeseries_multibatchanalysis.png - plot_legend: no - x-axis label: Time - y-axis label: unemployment_rate - linestyle: solid - marker: None - - -.. image:: ./images/P5_ts_unemployment_rate.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right - - - -**Example 6** (*For agent Eurostat, four sets, twenty runs each, one instance each, plotted in a single plot* ): - -main.yaml:: - - plot6: - timeseries: - agent: Eurostat - analysis: multiple_batch - variables: - var1: [monthly_output] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [0] - summary: mean - -plot.yaml:: - - plot6: - number_plots: one - plot_name: timeseries_multibatchanalysis.png - plot_legend: no - x-axis label: Time - y-axis label: monthly_output - linestyle: solid - marker: None - - -.. image:: ./images/P6_ts_monthly_output.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right - - -**Example 7** (*For agent Firm, four sets, twenty runs each, eighty instances, quantile values plotted in a single plot* ): - -main.yaml:: - - plot7: - timeseries: - agent: Firm - analysis: multiple_batch - variables: - var1: [price] - set: [10,13,16,17] - run: [range,[1,20]] - major: [range,[6020,12500,20]] - minor: [range,[1,80]] - summary: custom_quantile - quantile_values: - lower_quantile : 0.20 - upper_quantile : 0.80 - - -plot.yaml:: - - plot7: - number_plots: one - plot_name: ts_multibatch_analysis.png - plot_legend: yes - legend_location: best - x-axis label: Time - y-axis label: price - linestyle: solid - marker: None - fill_between_quartiles: yes - - -.. image:: ./images/P7_ts_price.png - :height: 100px - :width: 200 px - :scale: 50 % - :alt: alternate text - :align: right -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/TODO/merge_agentwise.py b/TODO/merge_agentwise.py deleted file mode 100755 index d627b0b..0000000 --- a/TODO/merge_agentwise.py +++ /dev/null @@ -1,159 +0,0 @@ - -import sqlite3, sys, glob, os, argparse, errno -import pandas as pd -from glob import glob as g -pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on -#TODO: serious fixes, when writing from multiple separate to agent based HDF5 files, minor axis is not copied correctly, i.e. only one of 20 axis is copied - -#TODO: frame gives error when trying to store in HDF store, due to unicode problem, you can either port this code to python3 or convert the frame to unicode - -DB_SUFFIX = '.h5' -agentname = "Eurostat" - -def write_hdf(fname, input_dbfolder): - set_run_name = fname[len(os.path.normpath(input_dbfolder))+3:-3] - print (set_run_name) - # Read the input HDF5 file in memory - store = pd.io.pytables.HDFStore(fname) - # Filtering out each agent types from the input HDF5 file - for key in store.keys(): - if agentname in key: - frame = store.select(key) - store_out[set_run_name] = frame - store.close() - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - reply = raw_input("Specified output directory already exists!! Delete existing directory named <<"+os.path.basename(d)+">> and all its contents? [y/n] ") - if reply in ['y', 'Y', 'yes']: - try: - os.system('rm -r '+ d) - "Directory named <<"+os.path.basename(d)+ ">> and all its contents deleted!!" - # Make new output folder - try: - os.makedirs(d) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - except: - error("- Could not delete directory <<" +os.path.basename(d)+">>. Directory may contain additional files, remove files manually and try again!") - else: - replytwo = raw_input("Continue & write output files inside existing directory: <<"+os.path.basename(d)+">> ? WARNING: This will overwrite old files having same name, if present in the folder! [y/n]: ") - if not replytwo in ['y', 'Y', 'yes']: - try: - print ("Please remove or rename the existing directory <<"+os.path.basename(d)+">> and try again, or choose a different directory for the output") - sys.exit() - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - else: - os.makedirs(d) - - -# Function to print out the error messages,if any, and exit -def error(mesg): - print (">>>>> (Error): %s" % mesg) - sys.exit(1) - - - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='db_hdf5_v2.py', description='Converts the SQLite database files to HDF5 files. For each db files, creates an equivalent HDF5 file.') - parser.add_argument('dbpath', help='Path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - - args = parser.parse_args() - - # Set input parameters - input_dbfolder = '' - input_dbfolder = args.dbpath[0] - if input_dbfolder == "./": - cwd = os.getcwd() - print ("- Python script and input csv files both inside the same folder <<" +cwd+">>. Expected at least one level of hierarchy!") - sys.exit(1) - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): - dir_list.append(dirpath) - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - - # Set output parameters - targetFolder = '' - if args.outpath: - targetFolder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - targetFolder = './output_'+os.path.basename(os.path.normpath(input_dbfolder)) # For output folder in the same folder where Python script is located. - #targetFolder = os.path.abspath(os.path.join(os.path.normpath(input_dbfolder), os.pardir))+'/output_'+os.path.basename(os.path.normpath(input_dbfolder)) # For output folder in the same folder where input folder is located - - - # Function call to check if the output folder already exists, and create if not present - dir_check(targetFolder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - #print (arg, end=' ') - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print() - else: - statusprint = lambda *a: None - - # Process each folder in the input directory - processed_folders = 0 - statusprint('\n- Total number of folders: '+ str(F)+'\n') - for i in range(N,len(dir_list)): - n = len(os.path.normpath(input_dbfolder)) + 2 - statusprint('- Started processing folder: '+os.path.basename(dir_list[i])) - if N == 1: - output_folder = targetFolder + dir_list[i][n:] - dir_check(output_folder) - else: - output_folder = targetFolder - - # Populate the list with all sql file names in the folder - db_file_list = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): - db_file_list.append(fname) - statusprint('- Total number of files within folder: '+ str(len(db_file_list))) - # Generate panels - processed_files =[] - #print (db_file_list) - outFileName = output_folder + '/' + agentname +'.h5' - store_out = pd.HDFStore(outFileName, 'w', chunksize = 500, complevel = 1, complib ='bzip2', fletcher32 = True) - for fname in db_file_list: - verboseprint('\n- Started processing: '+os.path.basename(fname)) - write_hdf(fname, input_dbfolder) - processed_files.append(fname) - percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), - store_out.close() - statusprint('- Finished processing folder: '+os.path.basename(dir_list[i])+'\n') - processed_folders = processed_folders+1 - f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - diff --git a/TODO/misc/replace_functions.py b/TODO/misc/replace_functions.py deleted file mode 100644 index e4278fb..0000000 --- a/TODO/misc/replace_functions.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -import sys, os - -FILE_SUFFIX = '.c' # Choose desired suffix, i.e. ".odt" or ".txt" or ".c" - -def process_string( string_in ): - def find_between( s, first, last ): - try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) - return s[start:end] - except ValueError: - return "" - string_out = find_between(string_in,"int","}") - return string_out - -def ProcessFile(output_folder): - #f_out = output_folder + '/' + os.path.basename(sys.argv[1])[:-2] + "_out" + FILE_SUFFIX - - f_a = open(sys.argv[1], 'r+') - input_lines_a = f_a.read() - print process_string( input_lines_a ) - f_a.close() - - f_b = open(sys.argv[2], 'r+') - input_lines_b = f_b.read() - print process_string( input_lines_b ) - f_b.close() - - # Replace the target string - replaced_lines_a = input_lines_a.replace(process_string(input_lines_a), process_string( input_lines_b )) - - f_out = open(sys.argv[1], 'w') - f_out.write(replaced_lines_a) - f_out.close() - - -if __name__ == "__main__": - output_folder = "./" - ProcessFile(output_folder) - - diff --git a/TODO/read_hdf.py b/TODO/read_hdf.py deleted file mode 100644 index 0bb7689..0000000 --- a/TODO/read_hdf.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -import sys, os, argparse -import numpy as np -import pandas as pd - -def process_hdf_keys( string_in ): - def find_between( s, first, last ): - try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) - return s[start:end] - except ValueError: - return "" - - tmp_string = string_in.replace('_run_', ',') - string_out = find_between(tmp_string,"/set_","_iters") - return list(map(int, string_out.split(','))) - -# Start with the main part - -store = pd.io.pytables.HDFStore('/home/susupta/Desktop/GitHub/Bank/Bank.h5') -d = pd.DataFrame() -for key in store.keys(): - sets_runs = process_hdf_keys(key) - s = sets_runs[0] - r = sets_runs[1] - # Opening Panel the particular set and run - pnl = store.select(key) - - # Converting panel to Dataframe - df = pnl.to_frame() - - # Adding two columns for set and run into the dataframe for two added level of indexing - df['set'] = s - df['run'] = r - df.set_index('run', append = True, inplace = True) - df.set_index('set', append = True, inplace = True) - d_i = df.reorder_levels(['set', 'run', 'major', 'minor']) - - # Adding each of the dataframe from panel into a main dataframe which has all the sets and runs - if d.empty: - d = d_i - else: - d = pd.concat([d,d_i], axis =0) - # Some tweak to get the multiindex working again for the main df - d.index = pd.MultiIndex.from_tuples(d.index,names=['set','run','major','minor']) - del df,d_i # Deleting sub df's for garbage collection -#print d['total_credit'] -store.close() diff --git a/TODO/visualisation/TODO.txt b/TODO/visualisation/TODO.txt deleted file mode 100644 index 47bb25c..0000000 --- a/TODO/visualisation/TODO.txt +++ /dev/null @@ -1,5 +0,0 @@ -set it such that if no range or data specified for set, run etc. in yaml file the whole dataset will be taken. - - - -Q: Starting a major architecture overhaul! diff --git a/TODO/visualisation/new_modular.py b/TODO/visualisation/new_modular.py deleted file mode 100644 index 0dd13b1..0000000 --- a/TODO/visualisation/new_modular.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python -import sys, os, argparse -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -import operator - -class A: - # Types of analysis - single, batch, parameter, agent = range(4) - # 0, 1, 2, 3 - -class M: - # Types of statistical summary - mean, median, upper_quartile, lower_quartile, custom_quantile, minimum, maximum = range(7) - # 0, 1, 2, 3, 4, 5 - -class SummaryStats(A): - def __init__(self, data, analysis_type): - self.__data = data - self.__analysis_type = analysis_type - #self.__method_type = method_type - #self.mean - - def mean(self): - single_analysis = lambda : self.__data.groupby(level = ['set','run','major']).mean().dropna() - batch_analysis = lambda : self.__data.groupby(level = ['set','run','major']).mean().dropna() - parameter_analysis = lambda : self.__data.groupby(level = ['set','major']).mean().dropna() - agent_analysis = lambda : self.__data.groupby(level=['major']).mean().dropna() - #error_input = lambda: [None, sys.stdout.write("Unexpected input value! Check again and retry... "), sys.exit(0)][0] - - #options = {0 : single_analysis, 1 : batch_analysis, 2 : parameter_analysis, 3 : agent_analysis, 4 : error_input} # error needs to be mapped later on - options = {A.single : single_analysis, A.batch : batch_analysis, A.parameter : parameter_analysis, A.agent : agent_analysis} - return options[self.__analysis_type]() - - def quantile(self, val): - single_analysis = lambda : self.__data.groupby(level = ['set','run','major']).quantile(val).dropna() - batch_analysis = lambda : self.__data.groupby(level = ['set','run','major']).quantile(val).dropna() - parameter_analysis = lambda : self.__data.groupby(level = ['set','major']).quantile(val).dropna() - agent_analysis = lambda : self.__data.groupby(level=['major']).quantile(val).dropna() - - options = {A.single : single_analysis, A.batch : batch_analysis, A.parameter : parameter_analysis, A.agent : agent_analysis} - return options[self.__analysis_type]() - - def custom_quantile(self): - N = raw_input("Enter desired number of Quantiles. 1 for single quantile and 2 for both Upper and Lower quantile values : ") - N = float(N) - if N == 1: - S = {} - Q = raw_input("Input the desired quantile. Input format: 0.XX, where XX is the numeric quantile value you want: ") - S['quantile'] = quantile(self, float(Q)) # N is the custom value for Quantile that is needed - elif N ==2: - S = dict.fromkeys(['lower_q', 'upper_q']) - Q1 = raw_input("Input the desired Lower quantile. Input format: 0.XX, where XX is the numeric quantile value you want: ") - Q2 = raw_input("Input the desired Upper quantile. Input format: 0.XX, where XX is the numeric quantile value you want: ") - S1 = quantile(self, float(Q1)) # Q is the custom value for Quantile that is needed - S['lower_q'] = S1 - S2 = quantile(self, float(Q2)) - S['upper_q'] = S2 - else: - print "Unidentified input values. Check input, and try again!" - sys.exit(1) - def maximum(self): - single_analysis = lambda : self.__data.groupby(level = ['set','run','major']).max().dropna() - batch_analysis = lambda : self.__data.groupby(level = ['set','run','major']).max().dropna() - parameter_analysis = lambda : self.__data.groupby(level = ['set','major']).max().dropna() - agent_analysis = lambda : self.__data.groupby(level=['major']).max().dropna() - options = {A.single : single_analysis, A.batch : batch_analysis, A.parameter : parameter_analysis, A.agent : agent_analysis} - return options[self.__analysis_type]() - - def minimum(self): - single_analysis = lambda : self.__data.groupby(level = ['set','run','major']).min().dropna() - batch_analysis = lambda : self.__data.groupby(level = ['set','run','major']).min().dropna() - parameter_analysis = lambda : self.__data.groupby(level = ['set','major']).min().dropna() - agent_analysis = lambda : self.__data.groupby(level=['major']).min().dropna() - - options = {A.single : single_analysis, A.batch : batch_analysis, A.parameter : parameter_analysis, A.agent : agent_analysis} - return options[self.__analysis_type]() - - - - - -def method(self): - options = {M.mean : mean, M.median : quantile(0.50), M.upper_quartile : quantile(0.75), M.lower_quartile : quantile(0.25), M.custom_quantile: custom_quantile, M.minimum : minimum, M.maximum : maximum} - return options[self.__analysis_type]() - - - -def Plots_main(df, M, A, NP): - N = 1000 # N is the number of points i.e 6020 to how much? i.e. 26000, generally 1000 - P = statistical_summary(df, M, A) # passing data, method to apply, and type of analysis - if NP == 1: - if M == 4: # Custom quartile needs two plots unlike the others so a separate method - idx = P.keys() - ################################################################################## - #for i in idx: - # d = P[i] - # y =[] - # for j in range(0,len(d),N): - # y.append(np.array(d[j:j+N])) - # for j in range(0,len(d)/N): - # x = np.linspace(0, N, N, endpoint=True) - # fig = str(i)+str(j) - # fig = plt.figure() - # - # fig.plot(x,y[j]) - # plt.hold(True) - #for i in idx: - # for j in range(0,len(d)/N): - # fig = str(i)+str(j) - # fig = plt.figure() - # plot_name = str(n)+".png" - # - # fig.savefig(plot_name, bbox_inches='tight') - # plt.clf() - ################################################################################### - else: - y =[] - for i in range(0,len(P),N): - y.append(np.array(P[i:i+N])) - for i in range(0,len(P)/N): - x = np.linspace(0, N, N, endpoint=True) - plt.plot(x,y[i]) - plot_name = str(i)+".png" - plt.savefig(plot_name, bbox_inches='tight') - # plt.show() # reset the plot, but gives output in display - # So, alternatively: - # plt.cla() # clear current axes - plt.clf() # clear current figure - # plt.close() # close the whole plot - - if NP == 0: - if M == 4: # Custom quartile needs two plots unlike the others so a separate method - idx = P.keys() - for i in idx: - d = P[i] - y =[] - for j in range(0,len(d),N): - y.append(np.array(d[j:j+N])) - for j in range(0,len(d)/N): - x = np.linspace(0, N, N, endpoint=True) - plt.plot(x,y[j]) - plt.hold(True) - plot_name = "summary_main.png" - plt.savefig(plot_name, bbox_inches='tight') - plt.show() - else: - y =[] - for i in range(0,len(P),N): - y.append(np.array(P[i:i+N])) - for i in range(0,len(P)/N): - x = np.linspace(0, N, N, endpoint=True) - plt.plot(x,y[i]) - plt.hold(True) - plt.savefig('summary_main.png', bbox_inches='tight') - plt.show() - - - - -#Main method -store = pd.io.pytables.HDFStore('/home/susupta/Desktop/GitHub/Bank/Bank.h5') -d = pd.DataFrame() -df_mean = [] -for key in store.keys(): - #print key - if len(key) == 18: - s = int(key[5:-12]) - r =int(key[11:-6]) - else: - s = int(key[5:-13]) - r =int(key[11:-6]) - pnl = store.select(key) - df = pnl.to_frame() - df['set'] = s - df['run'] = r - df.set_index('run', append = True, inplace = True) - df.set_index('set', append = True, inplace = True) - d_i = df.reorder_levels(['set', 'run', 'major', 'minor']) - if d.empty: - d = d_i - else: - d = pd.concat([d,d_i], axis =0) - d.index = pd.MultiIndex.from_tuples(d.index,names=['set','run','major','minor']) - del df,d_i - -filtered_df = d.iloc[(d.index.get_level_values('set') == 1) & (d.index.get_level_values('run') <= 2) & (d.index.get_level_values('major') <= 6100) & (d.index.get_level_values('minor') <= 2 )]['total_credit'].astype(float) - - -# instantiate a class with desired analysis type -P = SummaryStats(filtered_df, A.agent) -# then call the desired method -print P.mean() - -# instantiate a plot class with desired output (Single, Multiple) -Fig = Plot(P.mean(), NP.single) -# Calling the plot class instance with the desired kind of plot -Fig.timeseries() - -store.close() diff --git a/TODO/visualisation/src/__init__.py b/TODO/visualisation/src/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/TODO/visualisation/src/config.yaml b/TODO/visualisation/src/config.yaml deleted file mode 100644 index baa8614..0000000 --- a/TODO/visualisation/src/config.yaml +++ /dev/null @@ -1,111 +0,0 @@ -i/o: - input_path: '/home/susupta/Desktop/GitHub/Bank' - - input_files: - Bank: Bank.h5 # please name the key as the agent name - Eurostat: Eurostat.h5 - - output_path: '/home/susupta/Desktop/Fix_preprocess/visualize/src/output' - -plot1: - timeseries: - agent: Bank -# analysis: multiple_set - analysis: agent - variables: -# var1: [total_credit, '>[700]', '<[800]'] - var1: [total_credit] -# var2: [equity] - var2: [equity, '>[900]'] - set: [1] - run: [1,2] - major: [range,[6020,26000,20]] - minor: [1,5,7] - summary: custom_quantile - quantile_values: - lower_quantile : 0.50 - upper_quantile : 0.90 - - -#plot2: -# transform: -# agent: Bank -# analysis: multiple_set -# variables: -# var1: [total_credit] -# set: [1] -# run: [1] -# major: [range,[6020,6300,20]] -# minor: [1,5] -# summary: mean -# conditional_filtering: -# yes/no: no - - -#plot3: -# scatterplot: -# agent: Bank -# analysis: multiple_batch -# variables: -## var1: [total_credit, '>[700]', '<[800]'] -# var1: [total_credit] -## var2: [equity, '>[900]'] -# set: [1] -# run: [1,2] -# major: [range,[6020,26000,20]] -# minor: [1,5,7] -# summary: custom_quantile -# conditional_filtering: -# yes/no: yes -# condition_var: [total_credit, '>[700]'] -# operation_var: [total_credit, '>[700]', '<[800]'] - - -#plot3: -# histogram: -# agent: Bank -# analysis: multiple_batch -# variables: -## var1: [total_credit, '>[700]', '<[800]'] -# var1: [total_credit] -## var2: [equity, '>[900]'] -# set: [1] -# run: [1,2] -# major: [range,[6020,26000,20]] -# minor: [1,5,7] -# summary: mean -# conditional_filtering: -# yes/no: yes -# condition_var: [total_credit, '>[700]'] -# operation_var: [total_credit, '>[700]', '<[800]'] - - - -#plot4: -# timeseries: -# agent: Bank -# analysis: agent -# variables: -# var1: [total_credit] -# set: [1] -# run: [1,2] -# major: [range,[6020,6100,20]] -# minor: [range,[1,2]] -# summary: mean -# conditional_filtering: -# yes/no: no - - -#plot5: -# boxplot: -# agent: Bank -# analysis: multiple_set -# variables: -# var1: [total_credit] -# set: [1] -# run: [range,[1,2]] -# major: [range,[6020,26000,20]] -# minor: [1,5,7] -# conditional_filtering: -# yes/no: no - diff --git a/TODO/visualisation/src/parse_yaml.py b/TODO/visualisation/src/parse_yaml.py deleted file mode 100644 index a92af99..0000000 --- a/TODO/visualisation/src/parse_yaml.py +++ /dev/null @@ -1,39 +0,0 @@ -import sys -import yaml -from parameters import A, M, NP - -with open("config.yaml", 'r') as stream: - try: - #print(yaml.load(stream)) - d = yaml.load(stream) - #print a.keys() - except yaml.YAMLError as exc: - print(exc) - - -def process_parsed_values(d): - indices = ['set','run','major','minor'] - for i in indices: - if d.get(i) is not None: - if 'range' in str(d[i][0]): - x = d[i][1] - if len(x)<3: x.append(1) - d[i] = range(x[0],x[1],x[2]) - else: - print("Required parameters missing in the config.yaml file!") - sys.exit(1) - return d - - -for key in d.keys(): - d_plt = d[key] - param = process_parsed_values(d_plt) - #print param['plot properties']['number_plots'] - print param['analysis'] - -#print NP.single - - - - - diff --git a/TODO/visualisation/src/plot_config.yaml b/TODO/visualisation/src/plot_config.yaml deleted file mode 100644 index c937361..0000000 --- a/TODO/visualisation/src/plot_config.yaml +++ /dev/null @@ -1,47 +0,0 @@ -plot1: - number_plots: one - plot_name: timeseries_agentanalysis.png - plot_legend: yes - legend_location: best - legend_label: total_credit - x-axis label: some x label - y-axis label: some y label - linestyle: solid - marker: o - markerfacecolor: green - markersize: 1 - - - - -plot2: - number_plots: one - plot_name: second_one.png - l_lim: no - u_lim: no - tmin: no - tmax: no - plot_legend: no - legend_loc: out - legend_label: second_option - x-axis label: not yet done - y-axis label: some y label - linestyle: solid - -plot3: - number_plots: one - plot_name: third_one.png - l_lim: no - u_lim: no - tmin: no - tmax: no - plot_legend: no - legend_loc: out - legend_label: third_option - x-axis label: not yet done - y-axis label: some y label - linestyle: solid - -plot5: - number_plots: one - plot_name: sum_one.png diff --git a/TODO/visualisation/src/plots.py b/TODO/visualisation/src/plots.py deleted file mode 100644 index 0fbfffe..0000000 --- a/TODO/visualisation/src/plots.py +++ /dev/null @@ -1,711 +0,0 @@ -from parameters import NP, A, Plot_configuration -from summarystats import SummaryStats -import sys, os -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -plt.style.use('ggplot') -import matplotlib.cm as cm - - -class Plot(): - def __init__(self, idx, data, par_fpath): - self.idx = idx - self.__data = data - self.__P = Plot_configuration(par_fpath) - - def num_plot_mapper(self, val, obj): - one_plot = lambda: obj.one_output() - many_plot = lambda: obj.many_output() - options = {'one': one_plot, 'many': many_plot} - return options[val]() - - def timeseries(self, main_param, outpath): - T = Timeseries(self.idx, self.__data, self.__P, main_param, outpath) - self.num_plot_mapper(self.__P.num_plots(self.idx), T) - - def boxplot(self, main_param, outpath): - B = Boxplot(self.idx, self.__data, self.__P, main_param, outpath) - self.num_plot_mapper(self.__P.num_plots(self.idx), B) - - def scatterplot(self, main_param, outpath): - S = Scatterplot(self.idx, self.__data, self.__P, main_param, outpath) - self.num_plot_mapper(self.__P.num_plots(self.idx), S) - - def histogram(self, main_param, outpath): - H = Histogram(self.idx, self.__data, self.__P, main_param, outpath) - self.num_plot_mapper(self.__P.num_plots(self.idx), H) - - -class Timeseries(A): - - def __init__(self, idx, data, plt_config, main_param, outpath): - self.idx = idx - self.__data = data - self.__N = len(main_param['major']) - self.__analysistype = self.map_analysis(main_param['analysis']) - self.__P = plt_config - self.summary = main_param['summary'] - self.outpath = outpath + '/timeseries' - self.dir_check(self.outpath) - # self.ticklabels = main_param['major'] # only needed for tick label, which is temp atm - - def map_analysis(self, val): - analysis_values = {'agent': A.agent, 'multiple_run': A.multiple_run, 'multiple_batch': A.multiple_batch, 'multiple_set': A.multiple_set} - return analysis_values[val] - - # Function to check for existing directories, and create a new one if not present - def dir_check(self, d): - if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - else: - os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") - - def plot_line(self, ax, x, y, l_label, clr): - if self.__P.legend_label(self.idx) is None: - le_label = l_label - else: - le_label = self.__P.legend_label(self.idx) - out = ax.plot(x, y, linestyle=self.__P.linestyle(self.idx), marker=self.__P.marker(self.idx), - markerfacecolor=self.__P.markerfacecolor(self.idx), markersize=self.__P.markersize(self.idx), label=le_label, color = clr) - plt.xlabel(self.__P.x_label(self.idx)) - plt.ylabel(self.__P.y_label(self.idx)) - - if self.__P.legend(self.idx) is True: - ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) - return out - - - def one_output(self): - file_count = 0 - step = 1 - if self.summary == 'custom_quantile': - step = 2 - - for col in range(0, len(self.__data.columns), step): - if self.summary == 'custom_quantile': - dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() # one variable, one case at a time - else: - dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) - - legend_label = dframe.columns - fig, ax = plt.subplots() - - if self.__analysistype == A.agent: - minor_index = dframe.index.get_level_values('minor').unique() - for m in minor_index: - D = dframe.xs(int(m), level='minor') - if len(D.columns) == 2: - print("Quantile not possible for agent level analysis!") - sys.exit(1) - else: - y = [] - for l in range(0, len(D), self.__N): - y.append(np.array(D[l:l+self.__N])) - x = np.arange(1, self.__N+1) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(D)//self.__N))) - for r in range(0, len(y)): - clr = next(colors) - self.plot_line(ax, x, y[r], legend_label[0]+'_run_'+str(r)+str(m),clr) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0])+".png", bbox_inches='tight') - plt.close() - - else: - if len(dframe.columns) == 2: - y1 = [] - y2 = [] - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - - x = np.arange(1, self.__N+1) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) - for r in range(0, len(dframe)//self.__N): - clr = next(colors) - self.plot_line(ax, x, y1[r], legend_label[0]+'-'+str(r), clr) - self.plot_line(ax, x, y2[r], legend_label[1]+'-'+str(r), clr) - plt.fill_between(x, y1[r], y2[r], color=self.__P.fillcolor(self.idx), alpha=.5) - - - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - plt.close() - else: - y1 = [] - col_A = dframe[dframe.columns[0]] - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) - for r in range(0, len(dframe)//self.__N): - x = np.arange(1, self.__N+1) - clr = next(colors) - self.plot_line(ax, x, y1[r], legend_label[0] + " " + str(r), clr) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(str(self.outpath) + '/' + str(plot_name[:-4]) + '_' + str(legend_label[0]) + '.png', bbox_inches='tight') - plt.close() - file_count = file_count + 1 - - def many_output(self): - step = 1 - if self.summary == 'custom_quantile': - step = 2 - file_count = 0 - for col in range(0, len(self.__data.columns),step): - if self.summary == 'custom_quantile': - dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() # one variable, one case at a time - else: - dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) - legend_label = dframe.columns - - if self.__analysistype == A.agent: - print(" -Warning: too many plots will be produced !!! ") - minor_index = dframe.index.get_level_values('minor').unique() # get the index values for minor axis, which will later be used to sort the dataframe - for m in minor_index: - D = dframe.xs( int(m) , level='minor') - if len(D.columns) == 2: - print("Quantile not possible for agent level analysis") - sys.exit(1) - else: - count = 0 - for i in range(0,len(D),self.__N): - fig, ax = plt.subplots() - y = np.array(D[i:i+self.__N]) - x = np.arange(1, self.__N+1) - self.plot_line(ax, x, y, legend_label[0] + "_run_" + str(count) + "_instance_" + str(m)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0]) + "_run_" + str(count) + "_" + str(m) + ".png", bbox_inches='tight') - plt.close() - count = count + 1 - else: - if len(dframe.columns) == 2: - y1 = [] - y2 = [] - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - x = np.arange(1, self.__N+1) - for r in range(0, len(dframe)//self.__N): - fig, ax = plt.subplots() - self.plot_line(ax, x, y1[r], legend_label[0]+'_run_'+str(r)) - self.plot_line(ax, x, y2[r], legend_label[1]+'_run_'+str(r)) - plt.fill_between(x, y1[r], y2[r], color='k', alpha=.5) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - file_count = file_count + 1 - plt.close() - else: - y =[] - for i in range(0,len(dframe),self.__N): - y.append(np.array(dframe[i:i+self.__N])) - for s in range(0, len(dframe)//self.__N): - fig, ax = plt.subplots() - x = np.arange(1, self.__N+1) - self.plot_line(ax, x, y[s], legend_label[0] + "_" + str(s)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0]) + "_" + str(s) + ".png", bbox_inches='tight') - plt.close() - - -class Histogram(): - - def __init__(self, idx, data, plt_config, main_param, outpath): - self.idx = idx - self.__data = data - self.outpath = outpath + '/histogram' - self.dir_check(self.outpath) - #self.__N = len(main_param['major']) - self.__analysistype = self.map_analysis(main_param['analysis']) - self.__P = plt_config - self.summary = main_param['summary'] - if self.__analysistype == A.agent and self.summary == 'custom_quantile': - print(">> Quantile not possible for agent level analysis!") - sys.exit(1) - - def map_analysis(self, val): - analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} - return analysis_values[val] - - # Function to check for existing directories, and create a new one if not present - def dir_check(self, d): - if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - else: - os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") - - def plot_histogram(self, ax, data, label, colors, n_bins): - - if self.__P.legend_label(self.idx) is None: - le_label = label - else: - le_label = self.__P.legend_label(self.idx) - out = ax.hist(data, n_bins, histtype=self.__P.histtype(self.idx), stacked=self.__P.stacked(self.idx), normed=self.__P.norm(self.idx), fill=self.__P.fill(self.idx), color=colors, label=le_label) - if self.__P.legend(self.idx) is True: - plt.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) - if self.__P.plot_title(self.idx) is not None: - ax.set_title(self.__P.plot_title(self.idx)) - if self.__P.x_label(self.idx) is not None: - plt.xlabel(self.__P.x_label(self.idx)) - if self.__P.y_label(self.idx) is not None: - plt.ylabel(self.__P.y_label(self.idx)) - return out - - - def one_output(self): - file_count = 0 - step = 1 - if self.summary == 'custom_quantile': - step = 2 - - for col in range(0, len(self.__data.columns), step): # one variable, one case at a time - if self.summary == 'custom_quantile': - dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy().dropna() # hist method does not support NaN - else: - dframe = pd.DataFrame(self.__data[self.__data.columns[col]]).dropna() - - self.__N = len(dframe.index.get_level_values('major').unique()) - - legend_label = dframe.columns - if self.__analysistype == A.agent: - minor_index = dframe.index.get_level_values('minor').unique() - fig, ax = plt.subplots() - for m in minor_index: - D = dframe.xs(int(m), level='minor') - if len(D.columns) == 2: # TODO: this check done in class constructor, so no need - print("Quantile not possible for agent level analysis!") - sys.exit(1) - else: - y = [] - for l in range(0, len(D), self.__N): - y.append(np.array(D[l:l+self.__N])) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)/self.__N))) - for r in range(0, len(y)): - clr = next(colors) - self.plot_histogram(ax, y[r], legend_label[0]+'_run_'+str(r)+'_'+str(m), clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0])+".png", bbox_inches='tight') - plt.close() - - else: - if self.summary != 'full': - fig, ax = plt.subplots() - if len(dframe.columns) == 2: - y1 = [] - y2 = [] - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 4*len(dframe)/self.__N))) - for r in range(0, len(y1)): # TODO: y1 and y2 length must not be different, add a check - clr = next(colors) - self.plot_histogram(ax, y1[r], legend_label[0]+'_'+str(r), clr, self.__P.bins(self.idx)) - clr = next(colors) - self.plot_histogram(ax, y2[r], legend_label[1]+'_'+str(r), clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - plt.close() - else: - y1 = [] - col_A = dframe[dframe.columns[0]] - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) - for r in range(0, len(dframe)//self.__N): - clr = next(colors) - self.plot_histogram(ax, y1[r], legend_label[0]+'_'+str(r), clr, self.__P.bins(self.idx)) - - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0]) + ".png", bbox_inches='tight') - plt.close() - - else: # for the whole ensemble of data, if analysis is not agent level - fig, ax = plt.subplots() - if len(dframe.columns) == 2: ################TODO: two option not needed because when summary full, no sustom quantile possible - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 4))) - clr = next(colors) - self.plot_histogram(ax, col_A, legend_label[0], clr, self.__P.bins(self.idx)) - clr = next(colors) - self.plot_histogram(ax, col_B, legend_label[1], clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - plt.close() - else: - col_A = dframe[dframe.columns[0]] - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 1))) - clr = next(colors) - self.plot_histogram(ax, col_A, legend_label[0], clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0]) + ".png", bbox_inches='tight') - plt.close() - file_count = file_count + 1 - - def many_output(self): - - if (self.summary == 'full' and self.__analysistype != A.agent): - print(">> Multiple plots not possible for full ensemble of the data, select single plot option instead and retry!") - sys.exit(1) - - step = 1 - if self.summary == 'custom_quantile': - step = 2 - file_count = 0 - for col in range(0, len(self.__data.columns),step): - if self.summary == 'custom_quantile': - dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy().dropna() # one variable, one case at a time - else: - dframe = pd.DataFrame(self.__data[self.__data.columns[col]]).dropna() - - legend_label = dframe.columns - self.__N = len(dframe.index.get_level_values('major').unique()) - - if self.__analysistype == A.agent: - print(" -Warning: too many plots will be produced !!! ") - minor_index = dframe.index.get_level_values('minor').unique() # get the index values for minor axis, which will later be used to sort the dataframe - for m in minor_index: - D = dframe.xs( int(m) , level='minor') - if len(D.columns) == 2: - print("Quantile not possible for agent level analysis") - sys.exit(1) - else: - count = 0 - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) - for r in range(0,len(D),self.__N): - fig, ax = plt.subplots() - y = np.array(D[r:r+self.__N]) - clr = next(colors) - self.plot_histogram(ax, y, legend_label[0] + "_run_" + str(count) + "_instance_" + str(m), clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0]) + "_run_" + str(count) + "_" + str(m) + ".png", bbox_inches='tight') - plt.close() - count = count + 1 - else: - if len(dframe.columns) == 2: - y1 = [] - y2 = [] - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 4*len(dframe)//self.__N))) - for r in range(0, len(dframe)//self.__N): - fig, ax = plt.subplots() - clr = next(colors) - self.plot_histogram(ax, y1[r], legend_label[0] + "_run_" + str(r), clr, self.__P.bins(self.idx)) - clr = next(colors) - self.plot_histogram(ax, y2[r], legend_label[1] + "_run_" + str(r), clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - file_count = file_count + 1 - plt.close() - else: - y =[] - for i in range(0,len(dframe),self.__N): - y.append(np.array(dframe[i:i+self.__N])) - colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)/self.__N))) - for s in range(0, len(dframe)//self.__N): - fig, ax = plt.subplots() - clr = next(colors) - self.plot_histogram(ax, y[s], legend_label[0] + "_" + str(s), clr, self.__P.bins(self.idx)) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(legend_label[0]) + "_" + str(s) + ".png", bbox_inches='tight') - plt.close() - - -class Scatterplot(A): - - def __init__(self, idx, data, plt_config, main_param, outpath): - self.idx = idx - self.__data = data - self.__P = plt_config - self.outpath = outpath + '/scatterplot' - self.dir_check(self.outpath) - self.__N = len(main_param['major']) - self.__analysistype = self.map_analysis(main_param['analysis']) - self.delay = main_param['delay'] - self.summary = main_param['summary'] - - if self.delay is True: - if self.summary == 'custom_quantile': - print(">> Delay not supported for Quantiles! Adjust parameter and retry!") - sys.exit(1) - delayed_df = self.__data.shift(periods = 1, axis = 0) - delayed_df.rename(columns=lambda x: x+ "_delay", inplace=True) - D = pd.concat([self.__data, delayed_df], axis =1) - self.__data = D[list(sum(zip(delayed_df.columns, self.__data.columns), ()))] - - - def map_analysis(self, val): - analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} - return analysis_values[val] - - # Function to check for existing directories, and create a new one if not present - def dir_check(self, d): - if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - else: - os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") - - def plot_scatterplot(self, ax, x, y, l_label, x_label, y_label, clr): - if self.__P.legend_label(self.idx) is None: - le_label = l_label - else: - le_label = self.__P.legend_label(self.idx) - - out = ax.scatter(x, y, linestyle=self.__P.linestyle(self.idx), marker=self.__P.marker(self.idx), - facecolor=self.__P.facecolors(self.idx), label=le_label, color=clr) - - if self.__P.legend(self.idx) is True: - ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) - plt.xlabel(x_label) - plt.ylabel(y_label) - return out - - def one_output(self): - file_count = 0 - step = 2 - for col in range(0, len(self.__data.columns), step): - if len(self.__data.columns) < 2: - print(">> Problem with data! Either set delay to True, or specify atleast two variables to plot!") - sys.exit(1) - dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() - - if self.__analysistype == A.agent: - minor_index = dframe.index.get_level_values('minor').unique() - fig, ax = plt.subplots() # initialize figure - colors = iter(cm.rainbow(np.linspace(0, 1, len(dframe)//self.__N))) - for m in minor_index: - D = dframe.xs(int(m), level='minor') - legend_label = D.columns - if len(dframe.columns) != 2: - print(">> Something wrong with data, check and retry!") - sys.exit (1) - y1 = [] - y2 = [] - col_A = D[D.columns[0]] - col_B = D[D.columns[1]] - for i in range(0, len(D), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - for r in range(0, len(D)//self.__N): - clr = next(colors) - self.plot_scatterplot(ax, y1[r], y2[r], legend_label[0]+ ' vs '+legend_label[1]+' [inst'+str(m) + ' run' + str(r) + ']', legend_label[0], legend_label[1], clr ) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - plt.close() - - else: - if self.summary != 'full': - fig, ax = plt.subplots() # initialize figure - legend_label = dframe.columns - if len(dframe.columns) != 2: - print(">> Something wrong with data, check and retry!") - sys.exit (1) - y1 = [] - y2 = [] - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - colors = iter(cm.rainbow(np.linspace(0, 1, len(y1)))) - for r in range(0, len(dframe)//self.__N): - clr = next(colors) - self.plot_scatterplot(ax, y1[r], y2[r], legend_label[0]+' vs '+legend_label[1]+' [inst '+str(r) +']', legend_label[0], legend_label[1], clr) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - plt.close() - else: - fig, ax = plt.subplots() # initialize figure - legend_label = dframe.columns - if len(dframe.columns) != 2: - print(">> Something wrong with data, check and retry!") - sys.exit (1) - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - colors = iter(cm.rainbow(np.linspace(0, 1, 1))) - clr = next(colors) - self.plot_scatterplot(ax, col_A, col_B, legend_label[0]+' vs '+legend_label[1], legend_label[0], legend_label[1], clr) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(file_count) + ".png", bbox_inches='tight') - plt.close() - file_count = file_count + 1 - - def many_output(self): - file_count = 0 - step = 2 - for col in range(0, len(self.__data.columns), step): - if len(self.__data.columns) < 2: - print("Problem with data! Either set delay to True, or specify atleast two variables to plot!") - sys.exit(1) - dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() - - if self.__analysistype == A.agent: - minor_index = dframe.index.get_level_values('minor').unique() - colors = iter(cm.rainbow(np.linspace(0, 1, len(dframe)//self.__N))) - for m in minor_index: - D = dframe.xs(int(m), level='minor') - legend_label = D.columns - if len(dframe.columns) != 2: - print("Something wrong with data, check and retry!") - sys.exit (1) - y1 = [] - y2 = [] - col_A = D[D.columns[0]] - col_B = D[D.columns[1]] - for i in range(0, len(D), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - for r in range(0, len(D)//self.__N): - fig, ax = plt.subplots() - clr = next(colors) - self.plot_scatterplot(ax, y1[r], y2[r], legend_label[0]+ ' vs '+legend_label[1]+' [run'+str(r) + ' inst' + str(m) + ']', legend_label[0], legend_label[1], clr ) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4]+ '_'+ str(file_count) + '_run_' + str(r) + '_inst_' + str(m) + ".png", bbox_inches='tight') - plt.close() - else: - legend_label = dframe.columns - if len(dframe.columns) != 2: - print("Something wrong with data, check and retry!") - sys.exit (1) - y1 = [] - y2 = [] - col_A = dframe[dframe.columns[0]] - col_B = dframe[dframe.columns[1]] - for i in range(0, len(dframe), self.__N): - y1.append(np.array(col_A[i:i+self.__N])) - y2.append(np.array(col_B[i:i+self.__N])) - colors = iter(cm.rainbow(np.linspace(0, 1, len(y1)))) - for r in range(0, len(dframe)//self.__N): - fig, ax = plt.subplots() - clr = next(colors) - self.plot_scatterplot(ax, y1[r], y2[r], legend_label[0]+' vs '+legend_label[1]+' [inst '+str(r) +']', legend_label[0], legend_label[1], clr) - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4]+ '_'+ str(file_count) + '_'+ str(r) + ".png", bbox_inches='tight') - plt.close() - file_count = file_count + 1 - - -class Boxplot(A): - def __init__(self, idx, data, plt_config, main_param, outpath): - - self.idx = idx - self.__data = data - self.__P = plt_config - self.__main_param = main_param - self.outpath = outpath + '/boxplot' - self.dir_check(self.outpath) - self.__N = len(main_param['major']) - self.__analysistype = self.map_analysis(main_param['analysis']) - if self.__analysistype == A.agent: - print("Boxplot not possible for agent-level analysis!") - sys.exit(1) - - def map_analysis(self, val): - - analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} - return analysis_values[val] - - # Function to check for existing directories, and create a new one if not present - def dir_check(self, d): - if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - else: - os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") - - def process_boxplot_data(self, data): - - s = SummaryStats(data, self.__main_param) - box_df = pd.DataFrame() - #print s.mean().values - box_df['mean'] = s.mean() - box_df['median'] = s.median() - box_df['upper_quartile'] = s.upper_quartile() - box_df['lower_quartile'] = s.lower_quartile() - box_df['max'] = s.maximum() - box_df['min'] = s.minimum() - return box_df - - def plot_boxplot(self, ax, data, l_label): - - if self.__P.legend_label(self.idx) is None: - le_label = l_label - else: - le_label = self.__P.legend_label(self.idx) - - t_df = data.T - intervals = [] - pos = [] - N_bars = self.__P.numboxplotbars(self.idx) - count = 1 - for i in range(0,len(self.__main_param['major']), int(np.floor(len(self.__main_param['major'])//N_bars))): - intervals.append(self.__main_param['major'][i]) - pos.append(count) - count = count + 1 - #ax = t_df.boxplot(column = [100,250,500,750,900], positions =[1,2,3,4,5]) - ax = t_df.boxplot(column = intervals, positions =pos) - ax.set_title(le_label) - ax.set_xlabel(self.__P.x_label(self.idx)) - ax.set_ylabel(self.__P.y_label(self.idx)) - return ax - - def one_output(self): - - for col in range(0, len(self.__data.columns)): - dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) - fig, ax = plt.subplots() - if self.__analysistype == A.agent: # check done above, redundant, to remove - print("Boxplot not possible for agent-level analysis!") - sys.exit(1) - else: - col_A = dframe[dframe.columns[0]] - D = self.process_boxplot_data(col_A) - y = [] - for i in range(0, len(D), self.__N): - y.append(pd.DataFrame(D[i:i+self.__N])) - for r in range(0, len(D)//self.__N): - self.plot_boxplot(ax, y[r], self.__data.columns[col]) - - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(self.__data.columns[col]) + ".png", bbox_inches='tight') - plt.close() - - def many_output(self): - - for col in range(0, len(self.__data.columns)): - dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) - if self.__analysistype == A.agent: # check done above, redundant, to remove - print("Boxplot not possible for agent-level analysis!") - sys.exit(1) - else: - col_A = dframe[dframe.columns[0]] - D = self.process_boxplot_data(col_A) - y =[] - for i in range(0,len(D),self.__N): - y.append(pd.DataFrame(D[i:i+self.__N])) - for s in range(0, len(D)//self.__N): - fig, ax = plt.subplots() - self.plot_boxplot(ax, y[s], self.__data.columns[col]) - - plot_name = self.__P.plot_name(self.idx) - plt.savefig(self.outpath + '/' + plot_name[:-4] + "_" + str(self.__data.columns[col]) + "_" + str(s) + ".png", bbox_inches='tight') - plt.close() - - diff --git a/TODO/visualisation/src/plots1.py b/TODO/visualisation/src/plots1.py deleted file mode 100644 index 13bd8a4..0000000 --- a/TODO/visualisation/src/plots1.py +++ /dev/null @@ -1,269 +0,0 @@ -import sys -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -from parameters import NP, A -from summarystats import SummaryStats -from matplotlib.font_manager import FontProperties -#from timeseries import Timeseries -#from boxplot import Boxplot -#from histogram import Histogram - -class Parameter_mapper(): - def __init__(self, param): - self.__param = param - - def legend(self, key): - return self.__param[key]['plot_legend'] - - def legend_label(self, key): - return self.__param[key]['legend_label'] - - def plot_type(self, key): - return self.__param[key]['plot_type'] - - def num_plots(self, key): - return self.__param[key]['number_plots'] - - def y_label(self, key): - return self.__param[key]['y-axis label'] - - def x_label(self, key): - return self.__param[key]['x-axis label'] - - def plot_name(self, key): - return self.__param[key]['plot_name'] - - def llim(self, key): - return self.__param[key]['l_lim'] - - def ulim(self, key): - return self.__param[key]['u_lim'] - - def linestyle(self, key): - return self.__param[key]['linestyle'] - - -class Plot(NP): - def __init__(self, data, parameter): - self.__data = data - self.key = 'plot1' - self.__parameter = parameter - self.__param_map = Parameter_mapper(self.__parameter) - - def timeseries( self, n, analysis_type ): - T = Timeseries(self.__data, n, analysis_type, self.__parameter) - one_plot = lambda : T.one_output() - many_plot = lambda : T.many_output() - options = {'one' : one_plot, 'many' : many_plot} - return options[self.__param_map.num_plots(self.key)]() - - def histogram( self, n ): - H = Histogram(self.__data, num_plots, n) - one_plot = lambda : H.one_output() - many_plot = lambda : H.many_output() - options = {'one' : one_plot, 'many' : many_plot} - return options[self.__param_map.num_plots(self.key)]() - - - def boxplot(self): - B = Boxplot(self.__data,num_plots,self.__analysis_type) - one_plot = lambda : B.one_output() - many_plot = lambda : B.many_output() - options = {NP.one : one_plot, NP.many : many_plot} - return options[num_plots]() - - -class Timeseries(A): - - def __init__(self, data, n, a, parameter): - self.__data = data - self.__N = n - self.__analysistype = a - self.__parameter = parameter - self.__param_map = Parameter_mapper(self.__parameter) - self.key = 'plot1' - - def many_output(self): - print "yaha aayo many bhanne ma" - if self.__analysistype == A.agent: - print " -Warning: too many plots will be produced !!! " - count = 0 - minor_index = self.__data.index.get_level_values('minor').unique() # get the index values for minor axis, which will later be used to sort the dataframe - for i in minor_index: - D = self.__data.xs( int(i) , level='minor') - for i in range(0,len(D),self.__N): - y = np.array(D[i:i+self.__N]) - x = np.linspace(0, self.__N, self.__N, endpoint=True) - #plt.plot(x,y) - plt.plot(x,y,color = 'blue', linestyle=self.__param_map.x_label(self.key)['linestyle'], marker='o', markerfacecolor = 'green', markersize =4, label = self.__param_map.x_label(self.key)['legend_label']) - plot_name = self.__param_map.plot_name(self.key)[:-4]+str(count)+".png" - plt.savefig(plot_name, bbox_inches='tight') - plt.close() - count = count + 1 - else: - y =[] - for i in range(0,len(self.__data),self.__N): - y.append(np.array(self.__data[i:i+self.__N])) - count = 0 - for i in range(0,len(self.__data)/self.__N): - x = np.linspace(0, self.__N, self.__N, endpoint=True) - plt.plot(x,y[i]) - plot_name = self.__param_map.plot_name(self.key)[:-4]+str(count)+".png" - plt.savefig(plot_name, bbox_inches='tight') - count = count + 1 - - # plt.show() # reset the plot, but gives output in display - # So, alternatively: - # plt.cla() # clear current axes - plt.clf() # clear current figure - # plt.close() # close the whole plot - plt.close() - - def one_output(self): - print "yaha ni aayo, one bhanne ma" - if self.__analysistype == A.agent: - print " -Warning: too many lines will be printed in a single plot !!! " - minor_index = self.__data.index.get_level_values('minor').unique() # get the index values for minor axis, which will later be used to sort the dataframe - for i in minor_index: - D = self.__data.xs( int(i) , level='minor') - - for i in range(0,len(D),self.__N): - y = np.array(D[i:i+self.__N]) - x = np.linspace(0, self.__N, self.__N, endpoint=True) - plt.plot(x,y) - plt.hold(True) - plot_name = self.__param_map.plot_name(self.key) - plt.savefig(plot_name, bbox_inches='tight') - plt.close() - - else: - y =[] - for i in range(0,len(self.__data),self.__N): - y.append(np.array(self.__data[i:i+self.__N])) - count = 0 - for i in range(0,len(self.__data)/self.__N): - x = np.linspace(0, self.__N, self.__N, endpoint=True) - ######################################################################################################################## - #plt.plot(x,y[i], label = 'monthly output', color = 'green', linestyle='dashed', marker='o', markerfacecolor = 'blue', markersize =12) - plt.hold(True) - #line1, = plt.plot(x,y[i],color = 'green', linestyle='solid', marker='o', markerfacecolor = 'green', markersize =9) - #line2, = plt.plot(x,y[i],color = 'green', linestyle='dashed', marker='o', markerfacecolor = 'green', markersize =9) - plt.plot(x,y[i],color = 'blue', linestyle=self.__param_map.linestyle(self.key), marker='o', markerfacecolor = 'green', markersize =4, label = self.__param_map.legend_label(self.key)) - - #plt.plot(x,y[i],color = 'blue', linestyle='solid', marker='o', markerfacecolor = 'green', markersize =4, label = self.__param_map.legend_label(self.key)) - count = count + 1 - #plt.legend((line1, line2), ('label1', 'label2')) - #plt.legend(loc='best', fancybox=True, shadow=True) #alternative - - #fontP = FontProperties() - #fontP.set_size('small') - #plt.legend([line1], "title", prop = fontP) - - - ######################################################################################################################## - plt.legend(loc='best', fancybox=True, shadow=True) - plot_name = self.__param_map.plot_name(self.key) - plt.savefig(plot_name, bbox_inches='tight') - plt.close() - - -class Histogram(): - - def __init__(self, data, num_plots, n): - self.__data = data - self.__N = n - - def many_output(self): ####TODO### - y =[] - for i in range(0,len(self.__data),self.__N): - y.append(np.array(self.__data[i:i+self.__N])) - for i in range(0,len(self.__data)/self.__N): - hist, bins = np.histogram(y[i],bins = 50) - width = 0.7 * (bins[1] - bins[0]) - center = (bins[:-1] + bins[1:]) / 2 - plt.bar(center, hist, align='center', width=width) - #plt.show() - #plt.plot(x,y[i]) - plot_name = "histogram_"+str(i)+".png" - plt.savefig(plot_name, bbox_inches='tight') - - # plt.show() # reset the plot, but gives output in display - # So, alternatively: - # plt.cla() # clear current axes - plt.clf() # clear current figure - # plt.close() # close the whole plot - plt.close() - - def one_output(self): - self.__data.hist(bins = 50) - plt.savefig('histogram_main.png', bbox_inches='tight') - plt.close() - - -class Boxplot(NP, A): - def __init__(self, data, n, n_plots, a_type): - self.__data = data - self.__N = n - self.__a_type = a_type - self.__n_plots = n_plots - - def plot(self): - n_plot_values = {'one' : NP.one, 'many' : NP.many} - num_plots = n_plot_values[self.__n_plots] - one_plot = lambda : self.one_output() - many_plot = lambda : self.many_output() - options = {NP.one : one_plot, NP.many : many_plot} - return options[num_plots]() - - def f_analysis(self): - analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} - return analysis_values[self.__a_type] - - def one_output(self): - s = SummaryStats(self.__data, self.f_analysis() ) - box_df = pd.DataFrame() - box_df['mean'] = [x for sublist in s.mean().values for x in sublist] # [x for sublist in s.mean().values for x in sublist] done to flatten a 2D list to 1D so pandas accepts it - # box_df['mean'] = s.mean() # this was the old simpler method which did not work once the config file variables was turned to a hierarchy with filters (bug in df, see for new patches) - box_df['median'] = [x for sublist in s.median().values for x in sublist] - box_df['upper_quartile'] = [x for sublist in s.upper_quartile().values for x in sublist] - box_df['lower_quartile'] = [x for sublist in s.lower_quartile().values for x in sublist] - box_df['max'] = [x for sublist in s.maximum().values for x in sublist] - box_df['min'] = [x for sublist in s.minimum().values for x in sublist] - - bp = box_df.boxplot(column = ['min','median','mean','upper_quartile','lower_quartile','max'], positions =[1,3,4,5,2,6]) - # plt.hold(True) - plt.savefig('boxplot_main.png', bbox_inches='tight') - #plt.show() - plt.clf() - - - def many_output(self): - s = SummaryStats(self.__data, self.f_analysis() ) - box_df = pd.DataFrame() - box_df['mean'] = s.mean() - box_df['median'] = s.median() - box_df['upper_quartile'] = s.upper_quartile() - box_df['lower_quartile'] = s.lower_quartile() - box_df['max'] = s.maximum() - box_df['min'] = s.minimum() - - count = 0 - for i in range(0,len(box_df.index)/self.__N): - tmp_df = box_df[count:count+self.__N] - bp = tmp_df.boxplot(column = ['min','median','mean','upper_quartile','lower_quartile','max'], positions =[1,3,4,5,2,6]) - plot_name = "boxplot_"+str(i)+".png" - plt.savefig(plot_name, bbox_inches='tight') - plt.clf() - count = count + self.__N - plt.close() - - - - -###helpful pages: -# http://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.plot.html#matplotlib.axes.Axes.plot - -# legend outside the plot: -# http://stackoverflow.com/questions/4700614/how-to-put-the-legend-out-of-the-plot - diff --git a/TODO/visualisation/src/process_hdf_keys.py b/TODO/visualisation/src/process_hdf_keys.py deleted file mode 100644 index f3802d9..0000000 --- a/TODO/visualisation/src/process_hdf_keys.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -import sys, os, argparse -import numpy as np -import pandas as pd - -def process_file_keys( string_in ): - def find_between( s, first, last ): - try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) - return s[start:end] - except ValueError: - return "" - - tmp_string = string_in.replace('_run_', ',') - string_out = find_between(tmp_string,"/set_","_iters") - return list(map(int, string_out.split(','))) - - -if __name__ == "__main__": - # Opening the store to get the HDF file for Agent-type - store = pd.io.pytables.HDFStore('/home/susupta/Desktop/GitHub/Bank/Bank.h5') - # Going through sets and runs in the HDF file - for key in store.keys(): - print process_file_keys(key) diff --git a/TODO/visualisation/src/quantiles.py b/TODO/visualisation/src/quantiles.py deleted file mode 100644 index 5300e40..0000000 --- a/TODO/visualisation/src/quantiles.py +++ /dev/null @@ -1,24 +0,0 @@ - - -class quantiles(A,M,NP): - - N = 1000 - - def multiple_output(df): - # TODO - - - def single_output(df): - idx = df.keys() - for i in idx: - d = df[i] - y =[] - for j in range(0,len(d),N): - y.append(np.array(d[j:j+N])) - for j in range(0,len(d)/N): - x = np.linspace(0, N, N, endpoint=True) - plt.plot(x,y[j]) - plt.hold(True) - plot_name = "summary_main.png" - plt.savefig(plot_name, bbox_inches='tight') - plt.show() diff --git a/TODO/visualisation/src/scan_plot.py b/TODO/visualisation/src/scan_plot.py deleted file mode 100644 index 2dfd226..0000000 --- a/TODO/visualisation/src/scan_plot.py +++ /dev/null @@ -1,80 +0,0 @@ -import sys, os, argparse, yaml -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -# helper classes, import from same directory -from parameters import A, M, NP -from summarystats import SummaryStats -from plots1 import Plot, Boxplot - -def plt_timeseries( df, param ): - print param - for key in param.keys(): - print param[key]['plot_legend'] - - - P = SummaryStats(df, 2) - summary_type = {'mean': P.mean, 'median': P.median, 'upper_quartile': P.upper_quartile,'lower_quartile': P.lower_quartile,'custom_quantile': P.custom_quantile,'minimum': P.minimum,'maximum': P.maximum} - - n = 5 # number of datapoints for x-axis - #step = 4 # minor axis number of values - # instantiate a plot class with desired output (One, Many) - Fig = Plot(summary_type['mean'](), param) # first argument is one option selected from summary_type dict above - # Calling the plot class instance with the desired kind of plot - Fig.timeseries( n, 2) - - -def process_hdf_keys( string_in ): - def find_between( s, first, last ): - try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) - return s[start:end] - except ValueError: - return "" - tmp_string = string_in.replace('_run_', ',') - string_out = find_between(tmp_string,"/set_","_iters") - return list(map(int, string_out.split(','))) - - -with open("plot_config.yaml", 'r') as stream: - try: - param = yaml.load(stream) - except yaml.YAMLError as exc: - print(exc) - -for key, value in param.iteritems(): - print param[key]['plot_type'] - #print value - - - - - - -if __name__ == "__main__": - store = pd.io.pytables.HDFStore('/home/susupta/Desktop/GitHub/Bank/Bank.h5') - d = pd.DataFrame() - df_list =[] - for key in store.keys(): - sets_runs = process_hdf_keys(key) - s = sets_runs[0] - r = sets_runs[1] - pnl = store.select(key) - df = pnl.to_frame() - df['set'] = s - df['run'] = r - df.set_index('run', append = True, inplace = True) - df.set_index('set', append = True, inplace = True) - df_list.append(df.reorder_levels(['set', 'run', 'major', 'minor'])) - d = pd.concat(df_list) - del df_list - -filtered_df = d.iloc[(d.index.get_level_values('set') == 1) & (d.index.get_level_values('run') <= 2) & (d.index.get_level_values('major') <= 6100) & (d.index.get_level_values('minor') <= 3 )]['total_credit'].astype(float) - -plot_function = {'timeseries': plt_timeseries} - -plot_function['timeseries'](filtered_df, param) - -store.close() - diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/config/config.yaml b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/config/config.yaml new file mode 100644 index 0000000..3e7357a --- /dev/null +++ b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/config/config.yaml @@ -0,0 +1,634 @@ +i/o: + + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # set up your input_path for the resulting plots, it's relative unless it starts with a '/' + input_path: '/home/sander/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/h5_agentwise/' + + # set up your y_path for the resulting plots, it's relative unless it starts with a '/' + output_path: '/home/sander/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/Plots' + + input_files: + Agent: Agent.h5 + +#TIME SERIES - QUANTILE +## plot10: #Quantile not possible for agent level analysis! +## timeseries: +## agent: Agent +## analysis: agent +## variables: +## var1: [x,y] +## set: [range,[1,4]] +## run: [range,[1,2]] +## major: [range,[1,628,1]] +## minor: [range,[0,5]] +## summary: custom_quantile +## quantile_values: +## lower_percentile : 0.20 +## upper_percentile : 0.80 + +# plot11: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# plot12: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# plot13: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# #TIME SERIES - MEAN +# plot20: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot21: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot22: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot23: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean +##-------------------------------- +##Many plots +# plot24: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# set: [1] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot25: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot26: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot27: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# #TIME SERIES - FULL +# plot30: #Agent+Full does not exist +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot31: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot32: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot33: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# ## BOXPLOT - MEAN +# plot40: +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot41: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot42: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot43: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# ## BOXPLOT - FULL +# plot50: #Agent+Full does not exist +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot51: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot52: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot53: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +## SCATTERPLOT - MEAN +# ## One plot +# plot60: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] #this is index [0,5], should be using labels: [i].id +# summary: mean + +plot61: + scatterplot: + agent: Agent + analysis: multiple_run #shows summary across agents, for each run, each set + variables: + var1: [x,y] + delay: no + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[0,5]] + summary: mean + +plot62: + scatterplot: + agent: Agent + analysis: multiple_batch #shows summary across agents & runs, for each set + variables: + var1: [x,y] + delay: no + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[0,5]] + summary: mean + +plot63: + scatterplot: + agent: Agent + analysis: multiple_set #shows summary across agents & runs & sets + variables: + var1: [x,y] + delay: no + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[0,5]] + summary: mean +# ## -------------------------------- +# ## Many plots +# plot64: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot65: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot66: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot67: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# ## SCATTERPLOT - FULL +# plot70: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot71: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot72: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot73: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full +# ## -------------------------------- +# ## Many plots +# plot74: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot75: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot76: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot77: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# ## HISTOGRAM - MEAN +# plot80: +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean ## Note: the summary is mean value, over each level, gives hist of distribution of means + +# plot81: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot82: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot83: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# ## HISTOGRAM - FULL +# plot90: ## Agent+Full does not exist +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot91: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot92: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full # Note: here the histogram is over the entire distribution + +# plot93: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full # Note: here the histogram is over the entire distribution diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/config/plot_config.yaml b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/config/plot_config.yaml new file mode 100644 index 0000000..1e81baa --- /dev/null +++ b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/config/plot_config.yaml @@ -0,0 +1,564 @@ + +#TIME SERIES - QUANTILE +plot10: + number_plots: one + plot_name: p10_ts_quantile + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot11: + number_plots: one + plot_name: p11_ts_quantile_multi_run + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot12: + number_plots: one + plot_name: p12_ts_quantile_multi_batch + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot13: + number_plots: one + plot_name: p13_ts_quantile_multi_set + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +#TIME SERIES - MEAN +plot20: + number_plots: one + plot_name: p20_ts_mean_agent + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot21: + number_plots: one + plot_name: p21_ts_mean_multi_run + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot22: + number_plots: one + plot_name: p22_ts_mean_multi_batch + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot23: + number_plots: one + plot_name: p23_ts_mean_multi_set + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot24: + number_plots: many + plot_name: p24_ts_mean_agent_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + # xmin: -10.0 + # xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot25: + number_plots: many + plot_name: p25_ts_mean_multi_run_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot26: + number_plots: many + plot_name: p26_ts_mean_multi_batch_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot27: + number_plots: many + plot_name: p27_ts_mean_multi_set_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +#TIME SERIES - FULL +plot30: + number_plots: one + plot_name: p30_ts_full_agent + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot31: + number_plots: one + plot_name: p31_ts_full_multi_run + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot32: + number_plots: one + plot_name: p32_ts_full_multi_batch + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot33: + number_plots: one + plot_name: p33_ts_full_multi_set + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +# # BOXPLOT - MEAN +plot40: + number_plots: one + plot_name: p40_box_mean + plot_legend: yes + legend_location: upper right + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + +# BOXPLOT - FULL +plot50: + number_plots: one + plot_name: p50_box_full + plot_legend: yes + legend_location: upper right + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + bars: 50 + +#SCATTERPLOT - MEAN +plot60: + number_plots: one + plot_name: p60_scatter_mean_agent + plot_legend: no + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot61: + number_plots: one + plot_name: p61_scatter_mean_multi_run + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot62: + number_plots: one + plot_name: p62_scatter_mean_multi_batch + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot63: + number_plots: one + plot_name: p63_scatter_mean_multi_set + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot64: + number_plots: many + plot_name: p64_scatter_mean_agent_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot65: + number_plots: many + plot_name: p65_scatter_mean_multi_run_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot66: + number_plots: many + plot_name: p66_scatter_mean_multi_batch_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot67: + number_plots: many + plot_name: p67_scatter_mean_multi_set_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +#SCATTERPLOT - FULL +plot70: + number_plots: one + plot_name: p70_scatter_full_agent + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot71: + number_plots: one + plot_name: p71_scatter_full_multi_run + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot72: + number_plots: one + plot_name: p72_scatter_full_multi_batch + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot73: + number_plots: one + plot_name: p73_scatter_full_multi_set + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +# plot74: +# number_plots: many +# plot_name: p74_scatter_full_agent_many +# plot_legend: yes +# legend_location: upper right +# #legend_label: price vs. output +# ymin: -3.0 +# ymax: 4.0 +# xmin: -3.0 +# xmax: 4.0 +# marker: + + +plot75: + number_plots: many + plot_name: p75_scatter_full_multi_run_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot76: + number_plots: many + plot_name: p76_scatter_full_multi_batch_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot77: + number_plots: many + plot_name: p77_scatter_full_multi_set_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +#HISTOGRAM - MEAN +plot80: + number_plots: one + plot_name: p80_hist_mean_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: upper right + +plot81: + number_plots: one + plot_name: p81_hist_mean_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot82: + number_plots: one + plot_name: p82_hist_mean_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot83: + number_plots: one + plot_name: p83_hist_mean_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: upper right + +#HISTOGRAM - FULL +plot90: + number_plots: one + plot_name: p90_hist_full_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: upper right + +plot91: + number_plots: one + plot_name: p91_hist_full_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot92: + number_plots: one + plot_name: p92_hist_full_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot93: + number_plots: one + plot_name: p93_hist_full_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: upper right diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_1_run_1_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_1_run_1_iters.h5 new file mode 100644 index 0000000..be65a05 Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_1_run_1_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_1_run_2_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_1_run_2_iters.h5 new file mode 100644 index 0000000..16d2c68 Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_1_run_2_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_2_run_1_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_2_run_1_iters.h5 new file mode 100644 index 0000000..613ce52 Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_2_run_1_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_2_run_2_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_2_run_2_iters.h5 new file mode 100644 index 0000000..4e0147f Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_2_run_2_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_3_run_1_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_3_run_1_iters.h5 new file mode 100644 index 0000000..f468502 Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_3_run_1_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_3_run_2_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_3_run_2_iters.h5 new file mode 100644 index 0000000..bac880a Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_3_run_2_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_4_run_1_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_4_run_1_iters.h5 new file mode 100644 index 0000000..27075ec Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_4_run_1_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_4_run_2_iters.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_4_run_2_iters.h5 new file mode 100644 index 0000000..6acf243 Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5/set_4_run_2_iters.h5 differ diff --git a/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5agent/Agent.h5 b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5agent/Agent.h5 new file mode 100644 index 0000000..5e741ae Binary files /dev/null and b/data/data_conversion_scripts/test_output/db_h5/dataset_2_agents_6/h5agent/Agent.h5 differ diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/inputs/REAL_MODEL/xml.c b/data/replace_xml_writeout_functions/test_example/inputs/REAL_MODEL/xml.c similarity index 100% rename from src/data_conversion_scripts/misc/replace_xml_functions/test_example/inputs/REAL_MODEL/xml.c rename to data/replace_xml_writeout_functions/test_example/inputs/REAL_MODEL/xml.c diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/inputs/SHADOW_MODEL/shadow_model.xml b/data/replace_xml_writeout_functions/test_example/inputs/SHADOW_MODEL/shadow_model.xml similarity index 100% rename from src/data_conversion_scripts/misc/replace_xml_functions/test_example/inputs/SHADOW_MODEL/shadow_model.xml rename to data/replace_xml_writeout_functions/test_example/inputs/SHADOW_MODEL/shadow_model.xml diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/inputs/XML/shadow_xml.c b/data/replace_xml_writeout_functions/test_example/inputs/XML/shadow_xml.c similarity index 100% rename from src/data_conversion_scripts/misc/replace_xml_functions/test_example/inputs/XML/shadow_xml.c rename to data/replace_xml_writeout_functions/test_example/inputs/XML/shadow_xml.c diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/out.txt b/data/replace_xml_writeout_functions/test_example/out.txt similarity index 100% rename from src/data_conversion_scripts/misc/replace_xml_functions/test_example/out.txt rename to data/replace_xml_writeout_functions/test_example/out.txt diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/output/REAL_MODEL/xml.c b/data/replace_xml_writeout_functions/test_example/output/REAL_MODEL/xml.c similarity index 100% rename from src/data_conversion_scripts/misc/replace_xml_functions/test_example/output/REAL_MODEL/xml.c rename to data/replace_xml_writeout_functions/test_example/output/REAL_MODEL/xml.c diff --git a/data/tests/agentlist.txt b/data/tests/agentlist.txt new file mode 100644 index 0000000..b93e11e --- /dev/null +++ b/data/tests/agentlist.txt @@ -0,0 +1 @@ +Agent diff --git a/data/tests/db_h5_test.sh b/data/tests/db_h5_test.sh new file mode 100644 index 0000000..7e071b0 --- /dev/null +++ b/data/tests/db_h5_test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +export scriptpath=/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/src/data_conversion_scripts/db_hdf5 +export input=/media/sander/DataStorage-2/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6 +export output=/media/sander/DataStorage-2/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6 + +# Test v1 +# With compression, chunksize = 1000 +#python3 $scriptpath/db_hdf5_v1.py $input/db --chunksize 1000 --outpath $output/h5 -v -s -z 1 -c 1000 +# No compression, no chunksize +#python3 $scriptpath/db_hdf5_v1.py $input/db --outpath $input/h5 -v -s -z 0 + +# Test v2 +# With compression, chunksize = 1000 +#python3 $scriptpath/db_hdf5_v2.py $input/db --outpath $output/h5 -v -s -z 1 -c 1000 +# No compression, no chunksize +python3 $scriptpath/db_hdf5_v2.py $input/db --outpath $output/h5 -v -s -z 0 diff --git a/data/tests/flavis_test.sh b/data/tests/flavis_test.sh new file mode 100644 index 0000000..404c7be --- /dev/null +++ b/data/tests/flavis_test.sh @@ -0,0 +1,6 @@ +#!/bin/bash +export scriptpath=/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/src/visualisation_scripts +export configpath=/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/data/visualisation/models/circle/dataset_2_agents_6/config + +#This does not work? +python3 $scriptpath/main.py -p $configpath -v \ No newline at end of file diff --git a/data/tests/h5_agentwise_test.sh b/data/tests/h5_agentwise_test.sh new file mode 100644 index 0000000..4fff135 --- /dev/null +++ b/data/tests/h5_agentwise_test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +export scriptpath=/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/src/data_conversion_scripts/hdf_agentwise +export input=/media/sander/DataStorage-2/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/ +export output=/media/sander/DataStorage-2/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/ + +# With compression, chunksize = 1000 +#python3 $scriptpath/merge_hdf_agentwise.py $input/h5 agentlist.txt --outpath $output/merged_agent_wise -v -s -c 1000 -z 1 + +# No compression, no chunksize +python3 $scriptpath/merge_hdf_agentwise.py $input/h5 agentlist.txt --outpath $output/merged_agent_wise -v -s -z 0 + +# No output path +#python3 $scriptpath/merge_hdf_agentwise.py $input/h5 agentlist.txt -v -s -z 0 diff --git a/data/visualisation/CentralBank.h5 b/data/visualisation/CentralBank.h5 new file mode 100644 index 0000000..770929a Binary files /dev/null and b/data/visualisation/CentralBank.h5 differ diff --git a/data/visualisation/Eurostat.h5 b/data/visualisation/Eurostat.h5 new file mode 100644 index 0000000..b7f18f4 Binary files /dev/null and b/data/visualisation/Eurostat.h5 differ diff --git a/data/visualisation/Firm.h5 b/data/visualisation/Firm.h5 new file mode 100644 index 0000000..b91f1ff Binary files /dev/null and b/data/visualisation/Firm.h5 differ diff --git a/data/visualisation/models/circle/config/config.yaml b/data/visualisation/models/circle/config/config.yaml new file mode 100644 index 0000000..6d4f11f --- /dev/null +++ b/data/visualisation/models/circle/config/config.yaml @@ -0,0 +1,635 @@ +i/o: + + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # set up your input_path for the resulting plots, it's relative unless it starts with a '/' + #input_path: '/home/sander/scratch/svanderh/circle_test_model_data/its/dataset_1_agents_2/merged_agent_wise/' + input_path: '/home/sander/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/merged_agent_wise/' + + # set up your y_path for the resulting plots, it's relative unless it starts with a '/' + #output_path: '/home/sander/scratch/svanderh/circle_test_model_data/its/dataset_1_agents_2/Plots' + output_path: '/home/sander/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/Plots' + + input_files: + Agent: Agent.h5 + +#TIME SERIES - QUANTILE +## plot10: #Quantile not possible for agent level analysis! +## timeseries: +## agent: Agent +## analysis: agent +## variables: +## var1: [x,y] +## set: [range,[1,4]] +## run: [range,[1,2]] +## major: [range,[1,628,1]] +## minor: [range,[1,6]] +## summary: custom_quantile +## quantile_values: +## lower_percentile : 0.20 +## upper_percentile : 0.80 + +# plot11: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +plot12: + timeseries: + agent: Agent + analysis: multiple_batch + variables: + var1: [x,y] + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[1,6]] + summary: custom_quantile + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 + +# plot13: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# #TIME SERIES - MEAN +# plot20: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot21: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot22: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot23: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean +##-------------------------------- +##Many plots +# plot24: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# set: [1] +# run: [1] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot25: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot26: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot27: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# #TIME SERIES - FULL +# plot30: #Agent+Full does not exist +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot31: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot32: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot33: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# ## BOXPLOT - MEAN +# plot40: +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot41: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot42: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot43: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# ## BOXPLOT - FULL +# plot50: #Agent+Full does not exist +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot51: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot52: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot53: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# ## SCATTERPLOT - MEAN +# ## One plot +# plot60: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# delay: no +# set: [range,[1,2]] +# run: [1] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot61: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot62: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot63: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean +# ## -------------------------------- +# ## Many plots +# plot64: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [1] +# major: [range,[1,628,1]] +# minor: [2] +# summary: mean + +# plot65: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot66: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot67: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# ## SCATTERPLOT - FULL +# plot70: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot71: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot72: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot73: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full +# ## -------------------------------- +# ## Many plots +# plot74: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot75: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6,1]] +# summary: full + +# plot76: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# plot77: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full + +# ## HISTOGRAM - MEAN +# plot80: +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean ## Note: the summary is mean value, over each level, gives hist of distribution of means + +# plot81: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot82: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# plot83: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: mean + +# ## HISTOGRAM - FULL +# plot90: ## Agent+Full does not exist +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot91: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot92: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full # Note: here the histogram is over the entire distribution + +# plot93: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6]] +# summary: full # Note: here the histogram is over the entire distribution diff --git a/data/visualisation/models/circle/config/plot_config.yaml b/data/visualisation/models/circle/config/plot_config.yaml new file mode 100644 index 0000000..cc3537d --- /dev/null +++ b/data/visualisation/models/circle/config/plot_config.yaml @@ -0,0 +1,564 @@ + +#TIME SERIES - QUANTILE +plot10: + number_plots: one + plot_name: p10_ts_quantile + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot11: + number_plots: one + plot_name: p11_ts_quantile_multi_run + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot12: + number_plots: one + plot_name: p12_ts_quantile_multi_batch + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot13: + number_plots: one + plot_name: p13_ts_quantile_multi_set + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +#TIME SERIES - MEAN +plot20: + number_plots: one + plot_name: p20_ts_mean_agent + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot21: + number_plots: one + plot_name: p21_ts_mean_multi_run + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot22: + number_plots: one + plot_name: p22_ts_mean_multi_batch + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot23: + number_plots: one + plot_name: p23_ts_mean_multi_set + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot24: + number_plots: many + plot_name: p24_ts_mean_agent_many + plot_legend: yes + legend_location: best + # ymin: -10.0 + # ymax: 10.0 + # xmin: -10.0 + # xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot25: + number_plots: many + plot_name: p25_ts_mean_multi_run_many + plot_legend: yes + legend_location: best + ymin: -3.0 + ymax: 4.0 + #xmin: -3.0 + #xmax: 4.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot26: + number_plots: many + plot_name: p26_ts_mean_multi_batch_many + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot27: + number_plots: many + plot_name: p27_ts_mean_multi_set_many + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +#TIME SERIES - FULL +plot30: + number_plots: one + plot_name: p30_ts_full_agent + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot31: + number_plots: one + plot_name: p31_ts_full_multi_run + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot32: + number_plots: one + plot_name: p32_ts_full_multi_batch + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot33: + number_plots: one + plot_name: p33_ts_full_multi_set + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +# # BOXPLOT - MEAN +plot40: + number_plots: one + plot_name: p40_box_mean + plot_legend: yes + legend_location: best + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + +# BOXPLOT - FULL +plot50: + number_plots: one + plot_name: p50_box_full + plot_legend: yes + legend_location: best + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + bars: 50 + +#SCATTERPLOT - MEAN +plot60: + number_plots: one + plot_name: p60_scatter_mean_agent + plot_legend: no + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -1.0 + xmax: 2.0 + linestyle: solid + marker: + + +plot61: + number_plots: one + plot_name: p61_scatter_mean_multi_run + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot62: + number_plots: one + plot_name: p62_scatter_mean_multi_batch + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot63: + number_plots: one + plot_name: p63_scatter_mean_multi_set + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot64: + number_plots: many + plot_name: p64_scatter_mean_agent_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot65: + number_plots: many + plot_name: p65_scatter_mean_multi_run_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot66: + number_plots: many + plot_name: p66_scatter_mean_multi_batch_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot67: + number_plots: many + plot_name: p67_scatter_mean_multi_set_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +#SCATTERPLOT - FULL +plot70: + number_plots: one + plot_name: p70_scatter_full_agent + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot71: + number_plots: one + plot_name: p71_scatter_full_multi_run + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot72: + number_plots: one + plot_name: p72_scatter_full_multi_batch + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot73: + number_plots: one + plot_name: p73_scatter_full_multi_set + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot74: + number_plots: many + plot_name: p74_scatter_full_agent_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot75: + number_plots: many + plot_name: p75_scatter_full_multi_run_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot76: + number_plots: many + plot_name: p76_scatter_full_multi_batch_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot77: + number_plots: many + plot_name: p77_scatter_full_multi_set_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +#HISTOGRAM - MEAN +plot80: + number_plots: one + plot_name: p80_hist_mean_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: best + +plot81: + number_plots: one + plot_name: p81_hist_mean_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot82: + number_plots: one + plot_name: p82_hist_mean_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot83: + number_plots: one + plot_name: p83_hist_mean_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: best + +#HISTOGRAM - FULL +plot90: + number_plots: one + plot_name: p90_hist_full_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: best + +plot91: + number_plots: one + plot_name: p91_hist_full_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot92: + number_plots: one + plot_name: p92_hist_full_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot93: + number_plots: one + plot_name: p93_hist_full_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: best diff --git a/data/visualisation/models/circle/dataset_1_agents_2/config/config.yaml b/data/visualisation/models/circle/dataset_1_agents_2/config/config.yaml new file mode 100644 index 0000000..08fed4e --- /dev/null +++ b/data/visualisation/models/circle/dataset_1_agents_2/config/config.yaml @@ -0,0 +1,633 @@ +i/o: + + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # set up your input_path for the resulting plots, it's relative unless it starts with a '/' + input_path: 'data/visualisation/models/circle/dataset_1_agents_2/h5_agentwise/' + + # set up your y_path for the resulting plots, it's relative unless it starts with a '/' + output_path: 'data/visualisation/models/circle/dataset_1_agents_2/Plots' + + input_files: + Agent: Agent.h5 + +#TIME SERIES - QUANTILE +## plot10: #Quantile not possible for agent level analysis! +## timeseries: +## agent: Agent +## analysis: agent +## variables: +## var1: [x,y] +## set: [range,[1,2]] +## run: [range,[1,2]] +## major: [range,[1,628,1]] +## minor: [range,[1,2]] +## summary: custom_quantile +## quantile_values: +## lower_percentile : 0.20 +## upper_percentile : 0.80 + +# plot11: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +plot12: + timeseries: + agent: Agent + analysis: multiple_batch + variables: + var1: [x,y] + set: [range,[1,2]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[1,2]] + summary: custom_quantile + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 + +# plot13: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +#TIME SERIES - MEAN +# plot20: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot21: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot22: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot23: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean +##-------------------------------- +#Many plots +# plot24: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot25: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot26: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot27: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# #TIME SERIES - FULL +# plot30: #Agent+Full does not exist +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot31: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot32: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot33: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# ## BOXPLOT - MEAN +# plot40: +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot41: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot42: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot43: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# ## BOXPLOT - FULL +# plot50: #Agent+Full does not exist +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot51: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot52: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot53: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# ## SCATTERPLOT - MEAN +# ## One plot +# plot60: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot61: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot62: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot63: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean +# ## -------------------------------- +# ## Many plots +# plot64: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [2] +# summary: mean + +# plot65: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot66: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot67: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# ## SCATTERPLOT - FULL +# plot70: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot71: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot72: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot73: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full +# ## -------------------------------- +# ## Many plots +# plot74: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot75: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,6,1]] +# summary: full + +# plot76: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# plot77: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full + +# ## HISTOGRAM - MEAN +# plot80: +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean ## Note: the summary is mean value, over each level, gives hist of distribution of means + +# plot81: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot82: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# plot83: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: mean + +# ## HISTOGRAM - FULL +# plot90: ## Agent+Full does not exist +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot91: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot92: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full # Note: here the histogram is over the entire distribution + +# plot93: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,2]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[1,2]] +# summary: full # Note: here the histogram is over the entire distribution diff --git a/data/visualisation/models/circle/dataset_1_agents_2/config/plot_config.yaml b/data/visualisation/models/circle/dataset_1_agents_2/config/plot_config.yaml new file mode 100644 index 0000000..cc3537d --- /dev/null +++ b/data/visualisation/models/circle/dataset_1_agents_2/config/plot_config.yaml @@ -0,0 +1,564 @@ + +#TIME SERIES - QUANTILE +plot10: + number_plots: one + plot_name: p10_ts_quantile + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot11: + number_plots: one + plot_name: p11_ts_quantile_multi_run + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot12: + number_plots: one + plot_name: p12_ts_quantile_multi_batch + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot13: + number_plots: one + plot_name: p13_ts_quantile_multi_set + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +#TIME SERIES - MEAN +plot20: + number_plots: one + plot_name: p20_ts_mean_agent + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot21: + number_plots: one + plot_name: p21_ts_mean_multi_run + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot22: + number_plots: one + plot_name: p22_ts_mean_multi_batch + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot23: + number_plots: one + plot_name: p23_ts_mean_multi_set + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot24: + number_plots: many + plot_name: p24_ts_mean_agent_many + plot_legend: yes + legend_location: best + # ymin: -10.0 + # ymax: 10.0 + # xmin: -10.0 + # xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot25: + number_plots: many + plot_name: p25_ts_mean_multi_run_many + plot_legend: yes + legend_location: best + ymin: -3.0 + ymax: 4.0 + #xmin: -3.0 + #xmax: 4.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot26: + number_plots: many + plot_name: p26_ts_mean_multi_batch_many + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot27: + number_plots: many + plot_name: p27_ts_mean_multi_set_many + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +#TIME SERIES - FULL +plot30: + number_plots: one + plot_name: p30_ts_full_agent + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot31: + number_plots: one + plot_name: p31_ts_full_multi_run + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot32: + number_plots: one + plot_name: p32_ts_full_multi_batch + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot33: + number_plots: one + plot_name: p33_ts_full_multi_set + plot_legend: yes + legend_location: best + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +# # BOXPLOT - MEAN +plot40: + number_plots: one + plot_name: p40_box_mean + plot_legend: yes + legend_location: best + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + +# BOXPLOT - FULL +plot50: + number_plots: one + plot_name: p50_box_full + plot_legend: yes + legend_location: best + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + bars: 50 + +#SCATTERPLOT - MEAN +plot60: + number_plots: one + plot_name: p60_scatter_mean_agent + plot_legend: no + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -1.0 + xmax: 2.0 + linestyle: solid + marker: + + +plot61: + number_plots: one + plot_name: p61_scatter_mean_multi_run + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot62: + number_plots: one + plot_name: p62_scatter_mean_multi_batch + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot63: + number_plots: one + plot_name: p63_scatter_mean_multi_set + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot64: + number_plots: many + plot_name: p64_scatter_mean_agent_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot65: + number_plots: many + plot_name: p65_scatter_mean_multi_run_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot66: + number_plots: many + plot_name: p66_scatter_mean_multi_batch_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot67: + number_plots: many + plot_name: p67_scatter_mean_multi_set_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +#SCATTERPLOT - FULL +plot70: + number_plots: one + plot_name: p70_scatter_full_agent + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot71: + number_plots: one + plot_name: p71_scatter_full_multi_run + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot72: + number_plots: one + plot_name: p72_scatter_full_multi_batch + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot73: + number_plots: one + plot_name: p73_scatter_full_multi_set + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot74: + number_plots: many + plot_name: p74_scatter_full_agent_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot75: + number_plots: many + plot_name: p75_scatter_full_multi_run_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot76: + number_plots: many + plot_name: p76_scatter_full_multi_batch_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot77: + number_plots: many + plot_name: p77_scatter_full_multi_set_many + plot_legend: yes + legend_location: best + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +#HISTOGRAM - MEAN +plot80: + number_plots: one + plot_name: p80_hist_mean_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: best + +plot81: + number_plots: one + plot_name: p81_hist_mean_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot82: + number_plots: one + plot_name: p82_hist_mean_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot83: + number_plots: one + plot_name: p83_hist_mean_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: best + +#HISTOGRAM - FULL +plot90: + number_plots: one + plot_name: p90_hist_full_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: best + +plot91: + number_plots: one + plot_name: p91_hist_full_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot92: + number_plots: one + plot_name: p92_hist_full_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + +plot93: + number_plots: one + plot_name: p93_hist_full_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: best diff --git a/data/visualisation/models/circle/dataset_1_agents_2/db/set_1_run_1_iters.db b/data/visualisation/models/circle/dataset_1_agents_2/db/set_1_run_1_iters.db new file mode 100644 index 0000000..777d55a Binary files /dev/null and b/data/visualisation/models/circle/dataset_1_agents_2/db/set_1_run_1_iters.db differ diff --git a/data/visualisation/models/circle/dataset_1_agents_2/db/set_1_run_2_iters.db b/data/visualisation/models/circle/dataset_1_agents_2/db/set_1_run_2_iters.db new file mode 100644 index 0000000..777d55a Binary files /dev/null and b/data/visualisation/models/circle/dataset_1_agents_2/db/set_1_run_2_iters.db differ diff --git a/data/visualisation/models/circle/dataset_1_agents_2/db/set_2_run_1_iters.db b/data/visualisation/models/circle/dataset_1_agents_2/db/set_2_run_1_iters.db new file mode 100644 index 0000000..2dc96b5 Binary files /dev/null and b/data/visualisation/models/circle/dataset_1_agents_2/db/set_2_run_1_iters.db differ diff --git a/data/visualisation/models/circle/dataset_1_agents_2/db/set_2_run_2_iters.db b/data/visualisation/models/circle/dataset_1_agents_2/db/set_2_run_2_iters.db new file mode 100644 index 0000000..2dc96b5 Binary files /dev/null and b/data/visualisation/models/circle/dataset_1_agents_2/db/set_2_run_2_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/economic/FLAViz-scatterplot-agent-level-filenames-and-legend.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/economic/FLAViz-scatterplot-agent-level-filenames-and-legend.png new file mode 100644 index 0000000..f153a96 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/economic/FLAViz-scatterplot-agent-level-filenames-and-legend.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/economic/FLAViz-scatterplot-analysis-options.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/economic/FLAViz-scatterplot-analysis-options.png new file mode 100644 index 0000000..61aa137 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/economic/FLAViz-scatterplot-analysis-options.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-scatter-oneplot-summary-full-71-73.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-scatter-oneplot-summary-full-71-73.png new file mode 100644 index 0000000..6ea729b Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-scatter-oneplot-summary-full-71-73.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-scatter-oneplot-summary-mean-61-63.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-scatter-oneplot-summary-mean-61-63.png new file mode 100644 index 0000000..fbf7992 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-scatter-oneplot-summary-mean-61-63.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-manyplot-summary-mean-15-17.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-manyplot-summary-mean-15-17.png new file mode 100644 index 0000000..054923a Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-manyplot-summary-mean-15-17.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-agent-10.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-agent-10.png new file mode 100644 index 0000000..faeb835 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-agent-10.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-custom_quantile-31-33.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-custom_quantile-31-33.png new file mode 100644 index 0000000..1b55f6c Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-custom_quantile-31-33.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-11.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-11.png new file mode 100644 index 0000000..9a392a2 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-11.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-12.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-12.png new file mode 100644 index 0000000..6e075a8 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-12.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-13.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-13.png new file mode 100644 index 0000000..728dd14 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/github/test/FLAViz-timeseries-oneplot-summary-mean-13.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p80_hist_mean_agent_x.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p80_hist_mean_agent_x.png new file mode 100644 index 0000000..1a20579 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p80_hist_mean_agent_x.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p80_hist_mean_agent_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p80_hist_mean_agent_y.png new file mode 100644 index 0000000..019bd9a Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p80_hist_mean_agent_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p81_hist_mean_multiple_runs_x.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p81_hist_mean_multiple_runs_x.png new file mode 100644 index 0000000..6437716 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p81_hist_mean_multiple_runs_x.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p81_hist_mean_multiple_runs_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p81_hist_mean_multiple_runs_y.png new file mode 100644 index 0000000..51be2e7 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p81_hist_mean_multiple_runs_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p82_hist_mean_multiple_batches_x.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p82_hist_mean_multiple_batches_x.png new file mode 100644 index 0000000..f2f68df Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p82_hist_mean_multiple_batches_x.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p82_hist_mean_multiple_batches_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p82_hist_mean_multiple_batches_y.png new file mode 100644 index 0000000..440641c Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p82_hist_mean_multiple_batches_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p83_hist_mean_multiple_sets_x.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p83_hist_mean_multiple_sets_x.png new file mode 100644 index 0000000..96bde54 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p83_hist_mean_multiple_sets_x.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p83_hist_mean_multiple_sets_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p83_hist_mean_multiple_sets_y.png new file mode 100644 index 0000000..1b1beb1 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/histogram/p83_hist_mean_multiple_sets_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p60_scatter_mean_agent_Agent_x_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p60_scatter_mean_agent_Agent_x_y.png new file mode 100644 index 0000000..eb39b28 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p60_scatter_mean_agent_Agent_x_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p61_scatter_mean_multi_run_Agent_x_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p61_scatter_mean_multi_run_Agent_x_y.png new file mode 100644 index 0000000..64e86ec Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p61_scatter_mean_multi_run_Agent_x_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p62_scatter_mean_multi_batch_Agent_x_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p62_scatter_mean_multi_batch_Agent_x_y.png new file mode 100644 index 0000000..ab5395e Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p62_scatter_mean_multi_batch_Agent_x_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p63_scatter_mean_multi_set_Agent_x_y.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p63_scatter_mean_multi_set_Agent_x_y.png new file mode 100644 index 0000000..5adcde8 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p63_scatter_mean_multi_set_Agent_x_y.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p71_scatter_full_multi_run_Agent_x_y_full.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p71_scatter_full_multi_run_Agent_x_y_full.png new file mode 100644 index 0000000..72294f8 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p71_scatter_full_multi_run_Agent_x_y_full.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p72_scatter_full_multi_batch_Agent_x_y_full.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p72_scatter_full_multi_batch_Agent_x_y_full.png new file mode 100644 index 0000000..72294f8 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p72_scatter_full_multi_batch_Agent_x_y_full.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p73_scatter_full_multi_set_Agent_x_y_full.png b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p73_scatter_full_multi_set_Agent_x_y_full.png new file mode 100644 index 0000000..9fb1fcc Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/Plots/scatterplot/p73_scatter_full_multi_set_Agent_x_y_full.png differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/config/config.yaml b/data/visualisation/models/circle/dataset_2_agents_6/config/config.yaml new file mode 100644 index 0000000..960e378 --- /dev/null +++ b/data/visualisation/models/circle/dataset_2_agents_6/config/config.yaml @@ -0,0 +1,634 @@ +i/o: + + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # set up your input_path for the resulting plots, it's relative unless it starts with a '/' + input_path: '/media/sander/DataStorage-2/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/merged_agent_wise' + + # set up your y_path for the resulting plots, it's relative unless it starts with a '/' + output_path: '/media/sander/DataStorage-2/scratch/svanderh/circle_test_model_data/its/dataset_2_agents_6/merged_agent_wise/Plots' + + input_files: + Agent: Agent.h5 + +#TIME SERIES - QUANTILE +## plot10: #Quantile not possible for agent level analysis! +## timeseries: +## agent: Agent +## analysis: agent +## variables: +## var1: [x,y] +## set: [range,[1,4]] +## run: [range,[1,2]] +## major: [range,[1,628,1]] +## minor: [range,[0,5]] +## summary: custom_quantile +## quantile_values: +## lower_percentile : 0.20 +## upper_percentile : 0.80 + +# plot11: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# plot12: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# plot13: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: custom_quantile +# quantile_values: +# lower_percentile : 0.20 +# upper_percentile : 0.80 + +# #TIME SERIES - MEAN +# plot20: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot21: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot22: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot23: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean +##-------------------------------- +##Many plots +# plot24: +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# set: [1] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot25: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x] +# var2: [y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot26: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot27: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# #TIME SERIES - FULL +# plot30: #Agent+Full does not exist +# timeseries: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot31: +# timeseries: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot32: +# timeseries: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot33: +# timeseries: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# ## BOXPLOT - MEAN +# plot40: +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot41: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot42: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot43: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# ## BOXPLOT - FULL +# plot50: #Agent+Full does not exist +# boxplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot51: +# boxplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot52: +# boxplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot53: +# boxplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +## SCATTERPLOT - MEAN +# ## One plot +# plot60: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x] +# var2: [y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] #this is index [0,5], should be using labels: [i].id +# summary: mean + +plot61: + scatterplot: + agent: Agent + analysis: multiple_run #shows summary across agents, for each run, each set + variables: + var1: [x,y] + delay: no + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[0,5]] + summary: mean + +plot62: + scatterplot: + agent: Agent + analysis: multiple_batch #shows summary across agents & runs, for each set + variables: + var1: [x,y] + delay: no + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[0,5]] + summary: mean + +plot63: + scatterplot: + agent: Agent + analysis: multiple_set #shows summary across agents & runs & sets + variables: + var1: [x,y] + delay: no + set: [range,[1,4]] + run: [range,[1,2]] + major: [range,[1,628,1]] + minor: [range,[0,5]] + summary: mean +# ## -------------------------------- +# ## Many plots +# plot64: +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot65: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot66: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot67: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# ## SCATTERPLOT - FULL +# plot70: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot71: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot72: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot73: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full +# ## -------------------------------- +# ## Many plots +# plot74: ## Agent+Full does not exist +# scatterplot: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot75: +# scatterplot: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot76: +# scatterplot: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# plot77: +# scatterplot: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# delay: no +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full + +# ## HISTOGRAM - MEAN +# plot80: +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean ## Note: the summary is mean value, over each level, gives hist of distribution of means + +# plot81: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot82: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# plot83: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: mean + +# ## HISTOGRAM - FULL +# plot90: ## Agent+Full does not exist +# histogram: +# agent: Agent +# analysis: agent +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot91: +# histogram: +# agent: Agent +# analysis: multiple_run +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full ## Note: here the histogram is over the entire distribution + +# plot92: +# histogram: +# agent: Agent +# analysis: multiple_batch +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full # Note: here the histogram is over the entire distribution + +# plot93: +# histogram: +# agent: Agent +# analysis: multiple_set +# variables: +# var1: [x,y] +# set: [range,[1,4]] +# run: [range,[1,2]] +# major: [range,[1,628,1]] +# minor: [range,[0,5]] +# summary: full # Note: here the histogram is over the entire distribution diff --git a/data/visualisation/models/circle/dataset_2_agents_6/config/plot_config.yaml b/data/visualisation/models/circle/dataset_2_agents_6/config/plot_config.yaml new file mode 100644 index 0000000..1e81baa --- /dev/null +++ b/data/visualisation/models/circle/dataset_2_agents_6/config/plot_config.yaml @@ -0,0 +1,564 @@ + +#TIME SERIES - QUANTILE +plot10: + number_plots: one + plot_name: p10_ts_quantile + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot11: + number_plots: one + plot_name: p11_ts_quantile_multi_run + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot12: + number_plots: one + plot_name: p12_ts_quantile_multi_batch + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +plot13: + number_plots: one + plot_name: p13_ts_quantile_multi_set + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + #plot_format: pdf + +#TIME SERIES - MEAN +plot20: + number_plots: one + plot_name: p20_ts_mean_agent + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot21: + number_plots: one + plot_name: p21_ts_mean_multi_run + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot22: + number_plots: one + plot_name: p22_ts_mean_multi_batch + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot23: + number_plots: one + plot_name: p23_ts_mean_multi_set + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot24: + number_plots: many + plot_name: p24_ts_mean_agent_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + # xmin: -10.0 + # xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot25: + number_plots: many + plot_name: p25_ts_mean_multi_run_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot26: + number_plots: many + plot_name: p26_ts_mean_multi_batch_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +plot27: + number_plots: many + plot_name: p27_ts_mean_multi_set_many + plot_legend: yes + legend_location: upper right + ymin: 0.0 + ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + +#TIME SERIES - FULL +plot30: + number_plots: one + plot_name: p30_ts_full_agent + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot31: + number_plots: one + plot_name: p31_ts_full_multi_run + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot32: + number_plots: one + plot_name: p32_ts_full_multi_batch + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot33: + number_plots: one + plot_name: p33_ts_full_multi_set + plot_legend: yes + legend_location: upper right + #ymin: -10.0 + #ymax: 10.0 + #xmin: -10.0 + #xmax: 10.0 + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +# # BOXPLOT - MEAN +plot40: + number_plots: one + plot_name: p40_box_mean + plot_legend: yes + legend_location: upper right + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + +# BOXPLOT - FULL +plot50: + number_plots: one + plot_name: p50_box_full + plot_legend: yes + legend_location: upper right + #xlabel: Time + #ylabel: price + linestyle: solid + marker: None + greyscale: True + bars: 50 + +#SCATTERPLOT - MEAN +plot60: + number_plots: one + plot_name: p60_scatter_mean_agent + plot_legend: no + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot61: + number_plots: one + plot_name: p61_scatter_mean_multi_run + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot62: + number_plots: one + plot_name: p62_scatter_mean_multi_batch + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot63: + number_plots: one + plot_name: p63_scatter_mean_multi_set + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: 0.0 + ymax: 6.0 + xmin: 0.0 + xmax: 7.0 + linestyle: solid + marker: + + +plot64: + number_plots: many + plot_name: p64_scatter_mean_agent_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot65: + number_plots: many + plot_name: p65_scatter_mean_multi_run_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot66: + number_plots: many + plot_name: p66_scatter_mean_multi_batch_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +plot67: + number_plots: many + plot_name: p67_scatter_mean_multi_set_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + linestyle: solid + marker: + + +#SCATTERPLOT - FULL +plot70: + number_plots: one + plot_name: p70_scatter_full_agent + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot71: + number_plots: one + plot_name: p71_scatter_full_multi_run + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot72: + number_plots: one + plot_name: p72_scatter_full_multi_batch + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot73: + number_plots: one + plot_name: p73_scatter_full_multi_set + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +# plot74: +# number_plots: many +# plot_name: p74_scatter_full_agent_many +# plot_legend: yes +# legend_location: upper right +# #legend_label: price vs. output +# ymin: -3.0 +# ymax: 4.0 +# xmin: -3.0 +# xmax: 4.0 +# marker: + + +plot75: + number_plots: many + plot_name: p75_scatter_full_multi_run_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot76: + number_plots: many + plot_name: p76_scatter_full_multi_batch_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +plot77: + number_plots: many + plot_name: p77_scatter_full_multi_set_many + plot_legend: yes + legend_location: upper right + #legend_label: price vs. output + ymin: -3.0 + ymax: 4.0 + xmin: -3.0 + xmax: 4.0 + marker: + + +#HISTOGRAM - MEAN +plot80: + number_plots: one + plot_name: p80_hist_mean_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: upper right + +plot81: + number_plots: one + plot_name: p81_hist_mean_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot82: + number_plots: one + plot_name: p82_hist_mean_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot83: + number_plots: one + plot_name: p83_hist_mean_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: upper right + +#HISTOGRAM - FULL +plot90: + number_plots: one + plot_name: p90_hist_full_agent + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: True + legend_location: upper right + +plot91: + number_plots: one + plot_name: p91_hist_full_multiple_runs + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot92: + number_plots: one + plot_name: p92_hist_full_multiple_batches + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: upper right + +plot93: + number_plots: one + plot_name: p93_hist_full_multiple_sets + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: upper right diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_1_run_1_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_1_run_1_iters.db new file mode 100644 index 0000000..777d55a Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_1_run_1_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_1_run_2_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_1_run_2_iters.db new file mode 100644 index 0000000..2dc96b5 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_1_run_2_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_2_run_1_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_2_run_1_iters.db new file mode 100644 index 0000000..a032a49 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_2_run_1_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_2_run_2_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_2_run_2_iters.db new file mode 100644 index 0000000..e4db1c7 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_2_run_2_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_3_run_1_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_3_run_1_iters.db new file mode 100644 index 0000000..56ebeae Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_3_run_1_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_3_run_2_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_3_run_2_iters.db new file mode 100644 index 0000000..dde8c22 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_3_run_2_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_4_run_1_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_4_run_1_iters.db new file mode 100644 index 0000000..90a0aee Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_4_run_1_iters.db differ diff --git a/data/visualisation/models/circle/dataset_2_agents_6/db/set_4_run_2_iters.db b/data/visualisation/models/circle/dataset_2_agents_6/db/set_4_run_2_iters.db new file mode 100644 index 0000000..49f7290 Binary files /dev/null and b/data/visualisation/models/circle/dataset_2_agents_6/db/set_4_run_2_iters.db differ diff --git a/data/visualisation/models/circle/flame/agent_functions.c b/data/visualisation/models/circle/flame/agent_functions.c new file mode 100644 index 0000000..922a6dd --- /dev/null +++ b/data/visualisation/models/circle/flame/agent_functions.c @@ -0,0 +1,27 @@ +#include +//#include +#include "header.h" +#include "Agent_agent_header.h" + +#define PI 3.1415926535 +#define dx 1e-2 + +int set_xy(void) +{ + int t = iteration_loop; + + double s = fmod(t*dx,2*PI); + + A = ID; + + X = A + CONST_RADIUS*cos(s); + + B = RUN; + + Y = B + CONST_RADIUS*sin(s); + + printf("\nIT %d s %f ID %d CONST_RADIUS %f A %f B %f x %f y %f", t, s, ID, CONST_RADIUS, A, B, X, Y); + + return 0; +} + diff --git a/data/visualisation/models/circle/flame/circle_model.xml b/data/visualisation/models/circle/flame/circle_model.xml new file mode 100644 index 0000000..24c0712 --- /dev/null +++ b/data/visualisation/models/circle/flame/circle_model.xml @@ -0,0 +1,77 @@ + + Test Model - Xparser 0.17.1 Circle (x,y)-coordinates + Version 1.0, 06.06.2019 + Author: Sander van der Hoog (svdhoog@gmail.com) + + + + agent_functions.c + + + + int + set_no + + + + int + run + + + + double + const_radius + + + + double + const_b + + + + + + + Agent + + + + int + id + Agent ID. + + + double + x + Agent x coordinate. + + + double + y + Agent y coordinate. + + + double + a + Agent x-axis offset. + + + double + b + Agent y-axis offset. + + + + + set_xy + + start_Agent + end_Agent + + + + + + + + \ No newline at end of file diff --git a/data/visualisation/models/circle/its/0_circles_agents_2.xml b/data/visualisation/models/circle/its/0_circles_agents_2.xml new file mode 100644 index 0000000..15b667c --- /dev/null +++ b/data/visualisation/models/circle/its/0_circles_agents_2.xml @@ -0,0 +1,24 @@ + +0 + +0 +1.0 +0.0 + + +Agent + 1 + 0.0 + 0.0 + 1.0 + 0.0 + + +Agent + 2 + 0.0 + 0.0 + 2.0 + 0.0 + + \ No newline at end of file diff --git a/data/visualisation/models/circle/its/0_circles_agents_6.xml b/data/visualisation/models/circle/its/0_circles_agents_6.xml new file mode 100644 index 0000000..0b219b0 --- /dev/null +++ b/data/visualisation/models/circle/its/0_circles_agents_6.xml @@ -0,0 +1,56 @@ + +0 + +0 +1.0 +0.0 + + +Agent + 1 + 0.0 + 0.0 + 1.0 + 0.0 + + +Agent + 2 + 0.0 + 0.0 + 2.0 + 0.0 + + +Agent + 3 + 0.0 + 0.0 + 3.0 + 0.0 + + +Agent + 4 + 0.0 + 0.0 + 4.0 + 0.0 + + +Agent + 5 + 0.0 + 0.0 + 5.0 + 0.0 + + +Agent + 6 + 0.0 + 0.0 + 6.0 + 0.0 + + \ No newline at end of file diff --git a/data/visualisation/models/circle/readme.rst b/data/visualisation/models/circle/readme.rst new file mode 100644 index 0000000..8a4a28c --- /dev/null +++ b/data/visualisation/models/circle/readme.rst @@ -0,0 +1,187 @@ +Test data: Circles model +======= + +Purpose +------- + +To test the proper functionality of the library, this model creates test data that is both simple to understand, and predictable. + +The output plots should be clear in both the time series ``x(t)`` and scatter plot ``(x(t),y(t))`` dimensions. + + +Files: + +circle_model_ + +.. _circle_model: https://github.com/svdhoog/FLAViz/tree/master/data/visualisation/models/circle + +Dataset_1_ + +.. _Dataset_1: https://github.com/svdhoog/FLAViz/tree/master/data/visualisation/models/circle/dataset_1_agents_2/h5_agentwise + +Dataset_2_ + +.. _Dataset_2: https://github.com/svdhoog/FLAViz/tree/master/data/visualisation/models/circle/dataset_2_agents_6/h5_agentwise + + +Setup +------- + +This model produces test data with the following specifications: + +- Scatter plots in ``(x,y)``-space showing circles with a radius equal to the set number. +- The centre point of the circle is at ``(a,b)``. +- The x-axis horizontal shift ``a`` is equal to the agent ID. +- The y-axis vertical shift ``b`` is equal to the set number. + +The parametric equations for a circle with centre point ``(a,b)`` and radius ``r`` are given by: + + x(s) = a + r*cos(s) + + y(s) = b + r*sin(s), with s in [0,2*PI] + +For the time index ``s`` we use a transformation of the iteration counter ``t``, such that ``s`` remains in the interval ``[0,2 PI]``: + + s = (t.dx)mod(2*PI) + + dx = 1e-2 + +We let iterations ``t = 1...628`` such that ``100t=6.28`` which equals ``2 \PI``, approximately. + +Code +------- + +The C code for this model is: + + A = ID; + + X = A + CONST_RADIUS*cos(s); + + B = CONST_B; + + Y = B + CONST_RADIUS*sin(s); + +with + + CONST_RADIUS = set_no + + CONST_B = run_no + +Output Data Sets +------- + +**Data set 1:** + + Sets: 2 + + Runs: 1 + + Agents: 2 + +**Data set 2:** + + Sets: 4 + + Runs: 2 + + Agents: 6 + +Example plots +------- + +**Scatter plots** + +p60_scatter_mean_agent_Agent_x_y.png: + +.. image:: ./dataset_2_agents_6/Plots/scatterplot/p60_scatter_mean_agent_Agent_x_y.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + +p61_scatter_mean_multi_run_Agent_x_y.png: + +.. image:: ./dataset_2_agents_6/Plots/scatterplot/p61_scatter_mean_multi_run_Agent_x_y.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + +p62_scatter_mean_multi_batch_Agent_x_y.png: + +.. image:: ./dataset_2_agents_6/Plots/scatterplot/p62_scatter_mean_multi_batch_Agent_x_y.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + +**Time series** + +p12_ts_quantile_multi_batch_x.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p12_ts_quantile_multi_batch_x.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + +p12_ts_quantile_multi_batch_y.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p12_ts_quantile_multi_batch_y.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + +p20_ts_mean_agent_x.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p20_ts_mean_agent_x.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + +p24_ts_mean_agent_many_x_run_0_1.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p24_ts_mean_agent_many_x_run_0_1.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + + +p25_ts_mean_multi_run_many_x_0.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p25_ts_mean_multi_run_many_x_0.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + + +p26_ts_mean_multi_batch_many_x_0.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p26_ts_mean_multi_batch_many_x_0.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right + + +p27_ts_mean_multi_set_many_x_0.png: + +.. image:: ./dataset_2_agents_6/Plots/timeseries/p27_ts_mean_multi_set_many_x_0.png + :height: 100px + :width: 200 px + :scale: 50 % + :alt: + :align: right diff --git a/docs/data_conversion_scripts/FAQ.md b/docs/data_conversion_scripts/FAQ.md new file mode 100644 index 0000000..5d1d138 --- /dev/null +++ b/docs/data_conversion_scripts/FAQ.md @@ -0,0 +1,138 @@ +Notes on usage of the data conversion python scripts +==== + +Data storage format: HDF5 +==== + +The input data is stored in a ``HDF5`` container file (*.h5, .hdf5*) in a hierarchial form. + +Currently, the ``HDF5`` file is arranged as follows: + +- Each agent-type is contained in a separate HDF5 file, with the same name. + +- Each HDF5 file has a single hierarchy, with the ``agent-type`` as the root, and the ``set`` and ``runs`` as the branches. + +- the set and run branches each contain a DataFrame object, which contains a hierarchical index with these entries: ``iter``, ``id``. + +- the DataFrame is written to the HDF5 file with the help of ``Pytable`` module of python. + + +A HDF5 file as described above can be created from the SQLite db files by using the data processing scripts, included in the data processing directory + +*Note:* The ``SQLite`` db files follow this naming convention: ``set_*_run_*_iters.db`` + + +Data conversion Scripts +==== + +All scripts run on Python 3.4.3 and above. + +**XML_DB** + +These scripts convert ``XML`` files to ``SQLite`` database files. + +The script ``gendb.py`` creates SQLite db file from corresponding XML files in *many-to-one* fashion. So, for all xml files present in one input folder, one equivalent DB file is created in the output folder. +Example usage:: + + python3 gendb.py -m path_to_model_xml -x path_xml_files [-o output_path] + +Full usage example:: + + python3 gendb.py -m /home/user/my_model/model.xml -x /home/user/my_xml_data/ -o /home/user/my_db_data/ + +The script ``gendb_special.py`` creates SQLite db file from corresponding XML files in *many-to-one* fashion, and skips missing Agents. +Example usage:: + + python3 gendb_special.py -m path_to_model_xml -x path_xml_files [-o output_path] + +Full usage example:: + + python3 gendb_special.py -m /home/user/my_model/model.xml -x /home/user/my_xml_data/ -o /home/user/my_db_data/ + +Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: + + python gendb.py -h + python gendb_special.py -h + + +**DB_HDF5** + +- These scripts convert ``SQLite`` database files to ``HDF5`` files. +- There are two versions of the script currently, each having a slightly different functionality: + +#. The script ``db_hdf5_v1.py`` creates HDF5 files from corresponding SQLite db files in *many-to-one* fashion. + This means that for all db files present in some input folder, one corresponding big HDF5 file is created in the output folder. + For input folders containing more hierarchies, the **-r** flag can be passed which results in one HDF5 file for the contents of each subfolder. + +#. The script ``db_hdf5_v2.py`` creates HDF5 files from corresponding SQLite db files in *one-to-one* fashion. + This means that for each db file present in the input folder, a corresponding HDF5 file is created in the output folder. + To combine db files into a single HDF5 file, version 1 of the script named ``db_hdf5_v1.py`` should be used. + +#. For the visualization library, we recommend to use the following workflow: + + ``db files: set_*_run_*_iters.db`` -> ``db_hdf5_v2.py`` -> ``merge_hdf_agentwise.py`` -> ``visualisation_scripts/main.py`` + + - To convert db files to h5 files, the script ``db_hdf5_v2.py`` should be used. + - To produce one h5 file per agent type, use the script ``merge_hdf_agentwise.py`` + - The FLAViz visualization scripts read from the files ``Agent_name.h5`` + +- Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: + + python db_hdf5_v1.py -h + +- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. +- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. + These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out the option that is not preferred. Currently, the default is such that the output is created in the input folder. +- There is an option to switch on/off the internal compression in the HDFStore, by using the **-z 1** flag. +- In addition, the chunk size can be specified by the option **-c 1000**. + +**Notes:** + *There is no difference in the size of combined HDF5 (many-to-one generated by db_hdf5_v1.py) and separate HDF5 (one-to-one generated by db_hdf5_v2.py) files, even when compressing.* + *However, separate HDF5 files are easier to manipulate when restructuring the HDF5 into files per Agent-type (i.e. Bank.h5, Eurostat.h5 and so on).* + *Therefore, use of db_hdf5_v2.py is recommended for simplicity.* + + +**XML_HDF5** + +- This script converts ``XML`` files to ``HDF5`` files directly (without first converting XML to db and then to h5). + +- This script creates HDF5 files from corresponding XML files in *many-to-one* fashion. So, for all xml files present in one input folder, one equivalent HDF5 file is created in the output folder. + +- For input folders containing a folder hierarchy, given that a *-r* flag is passed, one HDF5 file is created for the contents of each subfolder. + +- Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: + + python xml_hdf5.py -h + +- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. +- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. + These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out + the option that is not preferred. Currently, the default is such that the output is created in the input folder. + + +**DB_XML** + +- This script converts ``SQLite`` Db files to ``XML`` files. +- This script creates XML files from corresponding DB files in *one-to-many* fashion. For each DB file, a folder is created and multiple corresponding XML files are created inside the folder, based on time samples. +- The script runs on Python 3.4 and above. Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: + + python genxml.py -h + +- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. +- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. + These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out + the option that is not preferred. Currently, the default is such that the output is created in the input folder. + + +**MERGE_HDF_AGENTWISE** + +- This script converts HDF5 files from ``set_*_run_*_iters.h5`` format to ``HDF5`` files per agent (**Eurostat.h5**, **Firm.h5** etc). +- A file **agentlist.txt** specifies which agent types should be considered as keys. +- For each HDF5 file (``set_*_run_*_iters.h5``) present in the input folder, the agents corresponding to an entry in the ``agentlist.txt`` file are filtered and stored inside a new agent-wise HDF5 file. +- Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: + + python merge_hdf_agentwise.py -h + +- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. +- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. + These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out the option that is not preferred. Currently, the default is such that the output is created in the input folder. diff --git a/docs/data_conversion_scripts/README.md b/docs/data_conversion_scripts/README.md new file mode 100644 index 0000000..8ac9317 --- /dev/null +++ b/docs/data_conversion_scripts/README.md @@ -0,0 +1,20 @@ +Data Conversion Scripts +==== +The scripts in this repository are for converting data produced by FLAME simulations. +Default output is to XML files; these scripts convert XML to SQL and HDF5. +Conversion from SQL to HDF5 is included as well. + +This repository contains Python scripts to convert between different data formats: + +- xml_db: conversion of XML to SQL +- db_hdf5: conversion of SQL to h5 +- db_xml: conversion of SQL to XML +- xml_hdf5: conversion of XML to h5 + +The repo also includes scripts to convert between different structures within the hdf5 file (``set_*_run_*_iters`` format to ``Agentname`` *[Eurostat, Bank etc]* ) format. + +- hdf_agentwise/merge_hdf_agentwise.py: conversion of h5 (per set, run) to h5 (per agent type) + +Usage +==== +The detailed documentation on how to use the module can be found on the */doc* folder in the [FAQ](FAQ.md). diff --git a/TODO/ETACE_architecture.xml b/docs/manual/FLAViz Workflow/FLAViz_architecture.xml similarity index 100% rename from TODO/ETACE_architecture.xml rename to docs/manual/FLAViz Workflow/FLAViz_architecture.xml diff --git a/TODO/workflow_diagram.pdf b/docs/manual/FLAViz Workflow/workflow_diagram.pdf similarity index 100% rename from TODO/workflow_diagram.pdf rename to docs/manual/FLAViz Workflow/workflow_diagram.pdf diff --git a/TODO/workflow_diagram.xml b/docs/manual/FLAViz Workflow/workflow_diagram.xml similarity index 100% rename from TODO/workflow_diagram.xml rename to docs/manual/FLAViz Workflow/workflow_diagram.xml diff --git a/docs/manual/tex/Benchmarks/Fig_100.png b/docs/manual/tex/Benchmarks/Fig_100.png new file mode 100644 index 0000000..78092ec Binary files /dev/null and b/docs/manual/tex/Benchmarks/Fig_100.png differ diff --git a/docs/manual/tex/Benchmarks/Fig_250.png b/docs/manual/tex/Benchmarks/Fig_250.png new file mode 100644 index 0000000..05135eb Binary files /dev/null and b/docs/manual/tex/Benchmarks/Fig_250.png differ diff --git a/docs/manual/tex/Benchmarks/Fig_513.png b/docs/manual/tex/Benchmarks/Fig_513.png new file mode 100644 index 0000000..f63dc4b Binary files /dev/null and b/docs/manual/tex/Benchmarks/Fig_513.png differ diff --git a/docs/manual/tex/Benchmarks/Fig_all.png b/docs/manual/tex/Benchmarks/Fig_all.png new file mode 100644 index 0000000..3761f6d Binary files /dev/null and b/docs/manual/tex/Benchmarks/Fig_all.png differ diff --git a/docs/manual/tex/FLAViz_Manual.pdf b/docs/manual/tex/FLAViz_Manual.pdf new file mode 100644 index 0000000..567ed43 Binary files /dev/null and b/docs/manual/tex/FLAViz_Manual.pdf differ diff --git a/docs/manual/tex/FLAViz_Manual.tex b/docs/manual/tex/FLAViz_Manual.tex new file mode 100644 index 0000000..46774dc --- /dev/null +++ b/docs/manual/tex/FLAViz_Manual.tex @@ -0,0 +1,1434 @@ +\documentclass[10pt,a4paper]{article} +\usepackage[utf8]{inputenc} +\usepackage{amsmath} +\usepackage{amsfonts} +\usepackage{amssymb} +\usepackage{graphicx, listings} +\usepackage{url,verbatim} +\usepackage{hyperref} +\usepackage{boxedminipage} + +\begin{document} + +\makeatletter + +\renewcommand*{\@fnsymbol}[1]{\ensuremath{\ifcase#1\or \star\or a\or b\or + \mathsection\or \mathparagraph\or \|\or \star\star\or \dagger\dagger + \or \ddagger\ddagger \else\@ctrerr\fi}} + +\makeatother + +\title{FLAViz:\\Flexible Large-scale Agent Visualization library\thanks{We gratefully acknowledge funding provided by the DFG Project Conquaire.}} + +\author{Sander van der Hoog\thanks{Email: svdhoog@gmail.com, Chair for Economic Theory and Computational Economics (ETACE), Dept. of Business Administration and Economics, Bielefeld University, Germany.}} + +\maketitle + + +\clearpage +\tableofcontents + +\clearpage +\section{Introduction} +This technical report describes the release of FLAViz v1.0, a new visualization library developed at the ETACE group at Bielefeld University (Economic Theory and Computational Economics). + +\subsection{Overview} + +The Flexible Large-scale Agent Visualization Library (FLAViz) is a software library developed specifically for data analysis and data visualization tasks on data that is produced by multi-agent, or agent-based, simulations (ABMs). Agent-based simulation models typically generate data across multiple dimensions, e.g. parameter sets, Monte Carlo replication runs, different agent types, multiple agent instances per type, many variables per agent, and time periods (iterations). This implies the data is structured as time series panel data sets. + +We specifically designed FLAViz having in mind the data generated by agent-based simulations developed in the FLAME framework, but in principle the output data from any ABM can be used, as long as the data adheres to our file specifications. + +FLAViz builds on Python pandas to deal with such high-dimensional time series panel data sets. +The data is stored as structured data using multiple hierarchical levels in the HDF5 file format. +This allows for proper data aggregation, filtering, selection, slicing, transformation, and visualization. +Various plots can be specified, e.g., time series, box plots, scatter plots, histograms, and delay plots. + +\paragraph{Headlines:} + +\begin{itemize} +\item FLAViz is part of the FLAME ecology, a set of tools to develop, simulate and analyse large-scale agent-based models.\footnote{FLAME stands for Flexible Large-scale Agent Modelling Environment. The FLAME website is at: \href{www.flame.ac.uk}{www.flame.ac.uk}, FLAME software can be downloaded from the GitHub repository at: \href{http://github/FLAME-HPC}{http://github/FLAME-HPC}.} + +\item FLAViz provides wrapper functionality around Python pandas to facilitate the data analysis and data visualization tasks that are typical for the time series data produced by agent-based model. +\end{itemize} + + +\paragraph{FLAViz provides the following features:} + +\begin{itemize} +\item Easy-to-configure \url{yaml} files + +\item Fully written in Python (support for both Python 2.7 and 3.6) + +\item Based on Python pandas (0.21.0) and matplotlib as standards for data analysis and visualization + +\item Data conversion from XML files to HDF5 files + +\item Data transformations of agent variables + +\item Data selection based on data values or indices (e.g., select all data at iteration $t=x$, or select all data for agent $ID=i$) + +\item Data filtering based on conditions (e.g., filter the selected data on the condition that agent variable X= 3.4.2) +python3-h5py (>= 2.2.1) +python3-matplotlib (>= 2.2.2) +python3-numexp (>= 2.6) +python3-numpy (>= 1.14.3) +python3-pandas (0.22.0 or >= 0.19.0) +python3-pyTables (>= 3.4) +python3-scipy (1.1.0) +python3-tk (>= 3.4.3) +python3-yaml (>= 3.12) +python3-lxml (4.2.1 or >= 3.3.3) +\end{verbatim} + +\item Data conversion scripts: ("recommended version", "minimal version") + +\begin{verbatim} +sqlite3 (>= 3.8.2) +python3-pandas (0.22.0 or >= 0.19.0) +python3-lxml (4.2.1 or >= 3.3.3) +\end{verbatim} +\end{itemize} + + +\paragraph{File listing:} + +\smallskip +Main files: +\begin{verbatim} +main.py +summarystats.py +plots.py +transform.py +\end{verbatim} + +Configuration files (user-created): +\begin{verbatim} +config.yaml +plot.yaml +transform.yaml +\end{verbatim} + +Other utility scripts (data transformations): +\begin{verbatim} +gendb.py - convert XML to SQLite db +db_hdf5_v1.py - convert db to h5 (per run) +db_hdf5_v2.py - convert db to h5 (per agent type) +genxml.py - convert db to XML +merge_hdf_agentwise.py - convert h5 (per run) to h5 (per agent type) +xml_hdf5.py - convert XML to h5 (per run) +\end{verbatim} + + +\subsection{Data Workflow \& Data Modules} + +\begin{itemize} +\item Fig. \ref{Fig: Simulation workflow} shows the simulation workflow, reflecting the order in which data is being generated, processes and visualized. This all relates to data in storage). + +\item Fig. \ref{Fig: Data processing} shows the data process flow, i.e. the order in which FLAViz uses the data. We adopt the ETL process. Even though the data extraction takes time, and it is common to execute the three phases in parallel, we have not yet done so. Our ETL is therefore still a sequential process. + +\item Fig. \ref{Fig: Package diagram} shows the FLAViz package diagram, showing the interconnections between the various modules. +\end{itemize} + +The software package consists of several modules, related to the different tasks, as shown in blue font in Fig. \ref{Fig: Package diagram}: + +\begin{itemize} +\item The \textbf{Main Module} performs that main tasks of data read-in from file, filtering the data by agent, variables, filtering conditions, and data selections. + +\item The \textbf{Summary Module} computes summary statistics. + +\item The \textbf{Transformation Module} performs data transformations. + +\item The \textbf{Plot Module} performs all tasks related to data visualization and table output. +\end{itemize} + + +\begin{figure} +\graphicspath{{./png/}} +% +\begin{boxedminipage}{14cm} +\centering\leavevmode +%\vspace{.5cm} +\includegraphics[scale=1]{simulation_workflow.pdf} +\end{boxedminipage} +\label{Fig: Simulation workflow} +\caption{\footnotesize Simulation flow.} +\end{figure} + +\begin{figure} +\centering\leavevmode +\graphicspath{{./png/}} +% +\begin{boxedminipage}{14cm} +\centering\leavevmode +\includegraphics[scale=.5]{data_processing.pdf} +\end{boxedminipage} +\label{Fig: Data processing} +\caption{\footnotesize Data processing flow. We follow the Extract-Transform-Load (ETL) process from data warehousing. Data extraction is where data is extracted from storage (left-most part, HDF5 files). Data transformation takes place in memory (middle part, using data frames and Python pandas). Data loading is where the transformed data is again written to storage (right-most part, different file formats).} +\end{figure} + + +\begin{figure}[Htp!] +\centering\leavevmode +\graphicspath{{./png/}} +% +\begin{boxedminipage}{16cm} +\centering\leavevmode +\includegraphics[scale=.4]{flavis_package_diagram.pdf} +\end{boxedminipage} +\label{Fig: Package diagram} +\caption{\footnotesize FLAViz package diagram.} +\end{figure} + + + +\subsection{File structure} + +This package consists of several Python scripts, located in the folder \url{/src}, dealing with different steps of the data visualisation and data transformation process: +\begin{itemize} +\item \url{main.py} : Contains code to read in the input data and primary configuration files (\url{config.yaml,transform.yaml}), filter the data based on filter conditions, and link the different Summary, Plot, and Transformation modules (see Fig. \ref{Fig: Package diagram}). + +\item \url{summarystats.py} : Takes in a Pandas dataframe, and computes the summary, and outputs the result as a Pandas dataframe. + +\item \url{plots.py}: Takes in a Pandas dataframe, and returns the necessary plots as specified in \url{config.yaml} and \url{plot.yaml}. + +\item \url{transform.py} : Takes in a Pandas dataframe, and returns/writes to a file the data transformations specified in \url{transform.yaml}. +\end{itemize} + +The configuration files are contained in a folder named \url{/config}, containing three configuration files: +\begin{itemize} +\item \url{config.yaml} : defines \url{i/o} \url{path}, \url{plot-types}, \url{agents} and specifies variables, filter options, summary. + +\item \url{plot.yaml} : defines plot properties i.e. \url{name, legends, line style, fill}, etc. + +\item \url{transform.yaml}: defines variables to transform, type of data transformations, and \url{i/o} info to store data in a file after transformations. +\end{itemize} + +Note: Further details on how to use the configuration (\url{yaml}) files can be found in Section \ref{Section:configuration} of this documentation. + +\subsection{Running the main module} + +To run the main executable, simply use: + +\begin{lstlisting} + $ python main.py path_to_configure_files +\end{lstlisting} + +where, \url{path_to_configure_files} is the path to the folder containing the \url{yaml} configuration files. + +\section{Data formats: Inputs and Outputs} +\label{Sec: Data formats} + +\subsection{FLAME output data} +Fig. \ref{Fig: Simulation workflow} shows the simulation workflow. The data is generated by a simulation model, and then stored to disk. The native data output format of FLAME simulations is XML, and the default data consists of all values of all memory variables for all agents. This is called a snapshot of the agent population. A simulation run consists of many iterations, and therefore FLAME produces many XML files, each file being a snapshot at the end of an iteration. The default setting of outputting a snapshot for every iteration can be ameliorated somewhat by selecting a subset of agent types, or a subset of memory variables to be outputted. Also a certain output frequency can be selected, which may help to further reduce the data deluge. + +After data storage of the per-iteration XML files, the files can be transformed into a single SQLite database file, to collect all data belonging to one simulation run. +A Python script that does the conversion from XML to SQLite is the \url{gendb.py} script. + +\subsubsection{Simulation data} +We adopted the following ontology to describe how we run simulations: + +\begin{itemize} +\item "Sets": a set reflects a model parameter setting. Each set differs from another set only in the parameter setting of the model. + +\item "Runs": a run is a replication, for a fixed parameter setting. Each run differs from other runs by the random seed only. The other initial conditions are kept exactly the same across runs. +\end{itemize} + +Thus, parameter variations are captured in "settings" or "sets". Each set reflects a different parametrization of the simulation model. +If we have any stochasticity in the model, we need to explore the statistical properties using the random seed for the Random Number Generator. +By default we use the RNG from the GNU/GSL library (a Mersenne Twister, mt19937, with a periodicity of $2^{19937} - 1$). + +For each set, we then perform multiple runs using different random seeds, producing different simulation output for each run. +These runs can be called Monte Carlo replication runs since the random seeds are themselves varied in a random fashion. +The seed is set randomly based on the system time at launch time, and then stored for later replication of the data if needed. + + +\subsubsection{Data heterogeneity} +The data is heterogeneous across several dimensions: + +\begin{itemize} +\item agent types: there can be many different agent types (e.g., household agents, firm agents, bank agents, etc.) + +\item agent instances: there can be a different number of individual agents (called agent instances) per agent type. + +\item agent memory variables: there can be a different number of memory variables per agent type (but all agents of the same type have the same set of memory variables, specified a priori in the model.xml file that fully specifies the model's structure). +\end{itemize} + +Due to this large data heterogeneity the file sizes may vary across simulations with the same model, even when using exactly the same input file, due to stochasticity. + + +\subsubsection{Data dimensions in the XML output} +(listed in the order in which data is being produced by FLAME): + +\begin{verbatim} +1 Sets +2 Runs +3 Iterations +4 Agent types +5 Agent instances per type +6 Variables +\end{verbatim} + +However, due to several conditions we have to impose on the data structure, the order in which data should be stored in the HDF5 file format differs from the order above. +There are two restrictions: + +\begin{itemize} +\item[1.] For the HDF5 file format it is important that the atomic data set at the lowest hierarchical level is a homgeneous data format. +This means that the choice of the 6 dimensions above requires us to choose those dimensions that remain invariant across all model simulations. +These dimensions are: 5 Agent instances, 3 Iterations, 6 Variables. These dimensions are invariant because we simulate the same model many times, and we do not change the model structure across simulations. +Therefore the number of variables per agent remains the same, the number of agent instances (individual agents) per agent type is constant, and the total number of iterations also remains constant across simulation runs. + +\item[2.] The 3D DataFrame format in Python pandas is specified as row-major. This means that the 3D data frame requires the largest dimension to be on the major axis (recall it has 3 axis; item, major and minor). +In our case, the largest dimension is the number of iterations, typically 1000 or higher. The other dimensions are the number of agent instances (~100), and the number of variables (~100). +\end{itemize} + +Therefore, we specify the 3D DataFrame as follows:\footnote{As of version 0.22.0, Python pandas is deprecating the data structure of the 3D DataFrame (or DataPanel), in favour of using the ndarray data structure. The scripts of FLAViz would have to be adapted in future releases to accommodate for this change.} +\begin{itemize} +\item item axis: agent instances +\item major axis: iterations (table rows of the DataFrame) +\item minor axis: variables (table columns of the DataFrame) +\end{itemize} + +\subsubsection{Data dimensions in the HDF5 file} + (listed in the order in which data is stored in the HDF5 files) + +\subsubsection*{Agent type: HDF5 filename (eg., Bank.h5)} + +Hierarchy inside the HDF5 file: +\begin{verbatim} +1 Sets: data group +2 Runs: data set inside data group +\end{verbatim} + +Hierarchy inside the pandas 3D DataPanel: +\begin{verbatim} +3 Agent instances +4 Iterations +5 Variables +\end{verbatim} + +\subsubsection{File format specifications} + +\paragraph{Step 1: XML output} + +This is the native format in which FLAME generates output. Each iteration produces an XML file that contains a full snapshot of all agents, and all agent memory variables. This can be a large file per iteration, so therefore FLAME also provides the possibility to only output one XML file at a certain frequency. This can be accomplished using the command: + +\begin{verbatim} +main -f freq +\end{verbatim} + +where \url{main} is the simulator executable and \url{freq} is an integer number specifying the periodicity with which output should be generated. +To further sub-sample the data outputted, it is possible to select only a subset of agents to output, or even to specify a \textbf{shadow model xml file} with only a subset of agent variables. Whatever method chosen, the data format is XML with fully tagged variables for each individual agent. Since the XML tags are rather verbose and redundant (of the format: \url{'value'}). This format can be reduced drastically by extracting the data to a more structured data format. + +\paragraph{Step 2: HDF5-per-run files} + +In this step we store data in one HDF5-per-run file per simulation run. + +Each simulation run in FLAME produces a set of XML files, one file per iteration. This set can be tansformed into a flat table format, with iteration number on the table rows, and all variables of an individual agent on the table columns. +This would produce many tables, as many as there are individual agents. To reduce this further, we structure the data using the following dimensions (see also above, Data dimensions in the HDF5 file): + +File hierarchy: +\begin{verbatim} +1 Set +2 Run +4 Agent type +\end{verbatim} + +Inside each HDF5-per-run file: +\begin{verbatim} +5 Agent instance +3 Iteration +6 Variable +\end{verbatim} + + +\paragraph{Step 3: HDF5-per-agent-type files} + +In this step we store data in one HDF5-per-agent-type file. this aggregates across all the files generated at the previous step 2 (all HDF5-per-run files). + +Structure: +\begin{verbatim} +HDF5 file: +Data group: "Sets" (parameter setting) +Data set: "Runs" (replications) stored as 3D DataPanel +\end{verbatim} + + +\begin{verbatim} +3D DataPanel: +item axis: agent instances +major axis (table rows): iterations +minor axis (table columns): variables +\end{verbatim} + + + +\subsection{Data preparation} + +The FLAViz library can only deal with input data that is stored in \emph{HDF5} container files (\url{.h5} or \url{.hdf5} file extension). The data inside such a file is stored using a hierarchical format. Currently, the HDF5 file has to be structured as follows: +\begin{itemize} +\item Data for each agent-type is stored in a separate HDF5 file, with the same name as the agent type name. + +\item Each HDF5 file has a single hierarchy, with the \emph{agent-type} as the root, and the \emph{set} and \emph{runs} as the sublevels. Inside the HDF5 hierarchy, the sets are stored as a 'data set', and the runs are stored as a data group. + +\item At the lowest level of the HDF5 hierarchy is a variable of type \emph{DataPanel}, which is Python pandas 3D data structure. This DataPanel itself contains three dimensions, also called \url{axes}: \url{major, minor}, and \url{items}. + +\item The 3D DataPanel data structure is written to the HDF5 file with the help of the \emph{PyTables} module in Python.\footnote{For performance reasons we use PyTables in 'fixed' write-only mode that does not allow to append data to the HDF5 file lateron. An alternative would be to use the 'append' mode, but our tests have shown that this incurs a considerable degradation in the I/O performance.} +\end{itemize} + +\paragraph{Note:} An HDF5 file as specified above can be created from raw data to the \url{h5} format, either by using the script \url{db_hdf5_v1.py} to convert SQLite \url{db} files, or by using the script \url{xml_hdf5.py} to convert the XML files directly. Both Python scripts are included in the data processing directory. + +\subsubsection{Data conversion scripts} + +The folder \url{data_conversion_scripts/} contains a set of Python scripts to translate between various file formats. +The conversion scripts currently included are: +\begin{verbatim} +gendb.py - convert XML to SQLite db +db_hdf5_v1.py - convert SQLite db to h5 (per run) +db_hdf5_v2.py - convert SQLite db to h5 (per agent type) +genxml.py - convert SQLite db to XML +merge_hdf_agentwise.py - convert h5 (per run) to h5 (per agent type) +xml_hdf5.py - convert XML to h5 (per run) +\end{verbatim} + + +\subsection{Usage of the conversion scripts} + +\paragraph{Note:} +To avoid any unwanted errors, it is imperative to name the SQLite database files using the following convention: + +\begin{verbatim} +set_s_run_r_iters.db +\end{verbatim} +where $s$ is the set number and $r$ is the run number. + +\subsubsection{Converting files: per-run SQLite db to per-run h5} +The script \url{db_hdf5_v1.py} converts a folder with \textbf{per-run} \url{db} files, creating a single HDF5 file in many-to-one fashion. +So for all the data content in the db files present in one folder, the data is combined into one equivalent HDF5 file in the output folder. +If the input folder contains a folder hierarchy, and if the recursive mode flag \url{-r} is set, then one HDF5 file is created for the contents of each subfolder. + +\begin{itemize} +\item Input files: \url{input_folder/set_*_run_*_iters.db} + +\item Output files: \url{output_folder/iters.h5} +\end{itemize} + +Example usage: +\begin{verbatim} +# Converts the SQLite database files to HDF5 files. +# Combines multiple db files into a single HDF5 file. +# Usage: db_hdf5_v1.py [-h] [-o OUTPATH] [-v] [-s] dbpath +# dbpath : Path to folder containing the .db files +# outpath : Path to folder for the output .h5 files +# '-o', '--outpath' : Path to the folder where the output is desired +# '-v', '--verbose' : Get the status of the intermediate processing steps +# '-s', '--status' : Get the total progress of the processing +# '-r', '--recursive': Recursively process all subfolders within the input folder + +# Example 1: non-recursive input folder +python3 db_hdf5_v1.py -o h5/ -v -s db_iters_files/ + +# Example 2: recursive input folder +python3 db_hdf5_v1.py -o h5/ -v -s -r db_iters_files/ +\end{verbatim} + +\subsubsection{Converting files: per-run SQLite db to per-run h5} +The script \url{db_hdf5_v2.py} converts \textbf{per-run} \url{db} files into the corresponding \textbf{per-run} \url{h5} format. By default, the script traverses a folder from its root into the entire folder hierarchy, searching for any \url{*.db} files in its subfolders, and converting these into their corresponding \url{*.h5} files. +\begin{itemize} +\item Input files: \url{set_*_run_*_iters.db} + +\item Output files: \url{set_*_run_*_iters.h5} +\end{itemize} + +Example usage: +\begin{verbatim} +# Convert iters.db to iters.h5 +# Usage: db_hdf5_v2.py [-h] [-o OUTPATH] [-v] [-s] dbpath +# -v: verbose mode, provides info on agent processed and overall progress +# in folder if multiple files present +# -s: status, provides info on file output status + +# Example: +python3 db_hdf5_v2.py -o h5_iters_files/ -v -s db_iters_files/ +\end{verbatim} + +\subsubsection{Converting files: h5 per-run to h5 per-agent-type} +The script \url{merge_hdf_agentwise.py} converts \textbf{per-run} h5 files to \textbf{per-agent-type} h5 files. +\begin{itemize} +\item Input files: + \begin{itemize} + \item \url{set_*_run_*_iters.h5}: per-run h5 files. + \item \url{agent_list.txt}: a plain text file with a list of agent types to be converted (one per line). + \end{itemize} +\item Output files: \url{Agentname.h5}: per-agent type h5 files. +\end{itemize} + +Example usage: +\begin{verbatim} +# Usage: merge_hdf_agentwise.py [-h] [-o OUTPATH] [-v] [-s] dbpath +# hdfpath: Path to folder containing the individual hdf files +# agentlist: File containing name of agent-types to process +# -o, --outpath: Path to the folder where the output is desired +# -v, --verbose: Get the status of the intermediate processing steps +# -s, --status: Get the total progress of the processing + +# Example: +python3 merge_hdf_agentwise.py -o h5_agentwise -s h5_iters_files/ agent_list.txt +\end{verbatim} + + +\subsubsection{Results} + +\begin{itemize} +\item Fig. \ref{Fig: h5 per run list} shows the output of the script \url{db_hdf5_v2.py}, which produces a list of per-run \url{h5} files. + +\item Fig. \ref{Fig: h5 per run contents} shows the contents of a per-run \url{h5} file. For the selected agent type \url{Bank}, the panel on the right-hand side shows the contents of the 3D DataPanel with the \url{major} axis (iterations), the \url{minor} axis (agent instances, 20 in this case), and the \url{item} axis (data content). Note that the data content is not clearly descernible because we have used the internal data compression for the HDF5 format. This clearly reduces the storage requirement, but makes the resulting \url{h5} files non-viewable. + +\item Fig. \ref{Fig: h5 per agent list} shows the output of the script \url{merge_hdf_agentwise.py}, which produces a list of per-agent-type \url{h5} files. + +\item Fig. \ref{Fig: h5 per agent contents} shows the contents of a per-agent-type \url{h5} file. The selected HDF5 data set corresponds to the agent type \url{Eurostat}, and the selected HDF5 data group is \url{set_1_run_1_iters}. The panel on the right-hand side shows the contents of the 3D DataPanel. The data in the \url{item} axis (third axis) is now decipherable, because in this case no internal data compression was used when writing to the HDF5 data format. +\end{itemize} + + +\begin{figure}[htb!] +\centering\leavevmode +\graphicspath{{./hdf5_snapshots/}} +% +%\begin{boxedminipage}{16cm} +\centering\leavevmode +\includegraphics[scale=.4]{Fig_0_list_of_separate_hdf5_files.png} +%\end{boxedminipage} +\label{Fig: h5 per run list} +\caption{\footnotesize Output of the script \url{db_hdf5_v2.py}: List of h5 files per run.} +\end{figure} + + +\begin{figure}[htb!] +\centering\leavevmode +\graphicspath{{./hdf5_snapshots/}} +% +%\begin{boxedminipage}{16cm} +\centering\leavevmode +\includegraphics[scale=.4]{Fig_1_individual_hdf5_corresponding_to_db_agent_1.png} +%\end{boxedminipage} +\label{Fig: h5 per run contents} +\caption{\footnotesize Output of the script \url{db_hdf5_v2.py}. The 3D DataPanel on the right shows the contents of the HDF5 data set \url{set_1_run_1_iters}, for the data group \url{Bank}. The data in the third axis of the 3D DataPanel, i.e. the \url{item} axis, is non-decipherable because we used the internal data compression of the HDF5 data format when writing the output to the file.} +\end{figure} + + +\begin{figure}[htb!] +\centering\leavevmode +\graphicspath{{./hdf5_snapshots/}} +% +%\begin{boxedminipage}{16cm} +\centering\leavevmode +\includegraphics[scale=.4]{Fig_4_list_of_hdf5_files_separated-per-agent-type.png} +%\end{boxedminipage} +\label{Fig: h5 per agent list} +\caption{\footnotesize Output of the script \url{merge_hdf_agentwise.py}: List of h5 files per agent type.} +\end{figure} + + +\begin{figure}[htb!] +\centering\leavevmode +\graphicspath{{./hdf5_snapshots/}} +% +%\begin{boxedminipage}{16cm} +\centering\leavevmode +\includegraphics[scale=.4]{Fig_3_hdf5_file_per_agent_showing_values.png} +%\end{boxedminipage} +\label{Fig: h5 per agent contents} +\caption{\footnotesize Output of the script \url{merge_hdf_agentwise.py}; contents of the HDF5 file for agent type Eurostat. The 3D DataPanel on the right shows the contents of the HDF5 data set \url{Eurostat}, for the data group \url{set_1_run_1_iters}. The data in the \url{item} axis (third axis) is decipherable, because in this case no internal data compression was used when writing to the HDF5 data format.} +\end{figure} + + +\subsection{Performance benchmarks} + +In this section we report on the performance of the process of transforming the SQLite database files (\url{*.db}) to HDF5 files (\url{*.h5}). +This process consists of three sub-processes. + +In the first sub-process (data preparation), the data structures required for the further processing of data lateron is being prepared. This involves parsing the model XML to read the agent types, their memory variables, and to allocate memory for the dataframe that will be filled in with values from the SQLite database files. This process is independent of the size of the problem. + +In the second sub-process (extracting and transforming the data), the SQLite database files are read-in from hard disk storage. This process is linear in the number of sets to process. Each file is extracted from storage, stored in memory, and then transformed into the native data format that is used by Python pandas, which is a flat table format, or dataframe. This dataframe can have an hierarchical index with 6 indices, as described in Section \ref{Sec: Data formats}. + +In the third sub-process (loading the data), the flat, hierarchical dataframe is transformed into the format of the 3D DataPanel, and then written out to HDF5 files. + +Fig. \ref{Fig: Performance} and Table \ref{Tab: Performance} show performance benchmarks for the Extract-Transform-Load process described above. We used sets of different sizes, i.e. the number of sets. Each set consisted of $1,0000$ runs of an ABM simulation, for $20,000$ iterations. Each ru takes aprox. 20 min, to complete, and yields a single output file of 64kb.\footnote{The data used for these performance benchmarks is available in the data publication van der Hoog, S. and Barde, S., 2017, Data for the paper: An empirical validation protocol for large-scale agent-based models, Bielefeld University. \url{https://10.4119/unibi/2908396}. The data for the 513 sets is contained in these files: +\url{calibration-mode-3-stage-1-sets-1-256-tarballs-part-1.tar.bz2} (7.47 GB) and \url{calibration-mode-3-stage-1-sets-257-513-tarballs-part-2.tar.bz2} (7.5 GB).} + +We considered only 3 cases, with 100, 250 and 513 sets, resp. For the largest case with 513 sets, the first 90 min./5 GB are used to read in the files from storage. After that, the transformation from flat 2D dataframes to the 3D DataPanel format takes about 260 min./145 GB. The final process of writing out the results to HDF5 takes approx. 50 min./50 GB. + +\begin{table} +\label{Tab: Performance} +\begin{tabular}{|c||cc||cc||cc|} +\hline \hline +Process: & \multicolumn{2}{|c|}{Preparation} & \multicolumn{2}{|c|}{Extract \& Transform} & \multicolumn{2}{|c|}{Load}\\ +\hline +No. sets & Time (min.) & RAM (GB) & Time (min.) & RAM (GB) & Time (min.) & RAM (GB)\\ +\hline +100 & 90 & 5 & 50 & 25 & 10 & 10\\ +\hline +250 & 100 & 5 & 130 & 70 & 25 & 25\\ +\hline +513 & 90 & 5 & 260 & 145 & 50 & 50\\ +\hline +\end{tabular} +\caption{Performance benchmarks for the data extraction from storage and the data processing in RAM memory.} +\end{table} + + + +\begin{figure}[htb!] +\centering\leavevmode +\graphicspath{{./Benchmarks/}} +% +%\begin{boxedminipage}{16cm} +\centering\leavevmode +\includegraphics[scale=.9]{Fig_all.png} +%\end{boxedminipage} +\label{Fig: Performance} +\caption{\footnotesize Performance benchmark for the ETL process.} +\end{figure} + +\clearpage +\section{FLAViz Configuration files} +\label{Section:configuration} + +There are three configuration files, which allow the user to input the necessary parameters for the program. The configuration files are specified in the \url{yaml} format that uses a hierarchical format which is not just for clarity but also for specifying functionality. Hence, it is important to abide by the indentation of the yaml files in order for them to be interpreted correctly. + +\paragraph{Note:} Any error in a \url{yaml} file might be caught by the exception handler, but indentation errors go unnoticed sometimes, which may result in undesired output. Hence, extra care is advised when formulating a configuration file. + +\subsection{File: config.yaml} + +\subsubsection*{Data input/output section} + +\url{i/o}: Specify the name of the repository root folder \url{repo_name} and input, output paths in the sub-hierarchies \url{input_path} and \url{output_path}. You can choose between an absolute path (\url{/path/to/your/files}) which always starts with a \url{'/'} or a relative path (\url{path/relative/to/project/folder}). It is necessary to set the name of the repository root folder to make use of relative paths. + +\begin{itemize} + \item The input path is specified as a full path in the sub-hierarchy \url{input_path}: +\end{itemize} + +\lstset{language=C,frame=single, basicstyle=\footnotesize} +\begin{lstlisting} + i/o: + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # for absolute input paths use: + input_path: '/path/to/FLAViz/data' + + # for relative input paths use: + input_path: 'data/visualisation' + + + # for absolute output paths use: + output_path: '/path/to/FLAViz/results' + + # for relative output paths use: + output_path: 'results' + + input_files: + CentralBank: CentralBank.h5 + Eurostat: Eurostat.h5 + Firm: Firm.h5 +\end{lstlisting} +\paragraph{Note:} The key name to the input path should correspond to the Agent-type (e.g., Bank, Eurostat, Firm etc.) + +\clearpage +\subsubsection*{Plot section} + +\begin{lstlisting} + plot1: + timeseries: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit] + var2: [equity] +\end{lstlisting} + +\begin{itemize} + \item \textbf{Plot-key} (e.g., \url{plot1}): Specify a key for the plot (mainly to keep track of the plot-number for other configuration files). Can be any string. + \item \textbf{Plot-type} (e.g., \url{timeseries}): Nested under \textbf{Plot-key} (here \url{plot1}), "Plot-type" specifies the type of plot desired. + \begin{itemize} + \item Possible values: \url{timeseries, boxplot, histogram, scatterplot}. + \end{itemize} +\end{itemize} + +\paragraph{Note [Exception]:} For data transformations, simply specify \url{transform} as the Plot-type, and it will perform the transform and store the new data items to a specified output file (no plots will be produced). +\begin{lstlisting} + plot1: + transform: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit] + var2: [equity] +\end{lstlisting} + +\begin{itemize} + \item \url{agent}: Name of the agent-type, nested under \textbf{Plot-type}. + + \item \url{analysis}: Type of analysis. Possible types: \url{agent, multiple_run, multiple_batch, multiple_set}. + + \item \url{variables}: Variables from the particular agent-type to be processed or visualized. The sub-hierarchy \url{var1, var2} etc. allows the input of multiple variables for any agent type. + The variable names can be inside a set of square brackets \url{[]} or simply inside a set of single-quotation marks \url{''}. +\end{itemize} + + +\subsubsection*{Conditional filtering} +The option to filter variables is based on filter conditions on the values, i.e. to retrieve only those values satisfying certain restrictions or are within a certain range. For conditional filtering, specify the variables as above, but with the filter conditions in addition. + +\begin{itemize} + \item Possible operator types are: \url{<,>,<=,>=,==}. + \item Simple or multiple filter conditions on a variable are possible. + \item Filtering on multiple variables can be specified. +\end{itemize} + +\begin{lstlisting}[caption={Filtering examples using single and multiple filter conditions.}] + var1: [variable name, 'operator[value]'] + e.g. + var1: [total_credit, '>[700]'] + # Select only values of total credit greater than 700. + + var2: [variable name, 'operator1[value]','operator2[value]'] + e.g. + var2: [equity, '>[700]', '<[1500]'] + # Select only values of total credit between 700 and 1500. +\end{lstlisting} + +\begin{lstlisting}[caption={Filtering example using multiple variables.}] + plot1: + timeseries: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit, '>[700]'] + var2: [equity, '>[700]', '<[800]'] +\end{lstlisting} + +\subsubsection*{Data selection} +For the \textbf{sets}, \textbf{runs}, \textbf{major} and \textbf{minor} axes, the data selection can be specified either as a ranges or as a list:\footnote{The third axis of the 3D DataPanel, the \url{item} axis, is deliberately left out of this list for data selections, since it is used to store the variables. These can be selected separately under the \url{'variables'} section in the \url{config.yaml} file.} +\begin{itemize} +\item \url{set} : Specify the sets to process. Input can be an explicit list, or (esp. for long lists) a custom way is to specify a range of values. + +\item \url{run} : Specify the runs to process. Syntax is similar to \url{sets} above. + +\item \url{major} : Specify the values from the major axis (iterations). Syntax similar to \url{sets} above. + +\item \url{minor} : Specify the values from the minor axis (agent instances). Syntax similar to \url{sets} above. +\end{itemize} + +Example of selecting sets using an explicit list of values: +\begin{lstlisting} + set: [val(1),val(2),...,val(N)] + e.g. + set: [1,2] +\end{lstlisting} + +Example of selecting sets using a range of values (list with values from 1 to 10 with a step size of 2): +\begin{lstlisting} + set: [range,[val(1),val(N),stepsize]] + e.g. + set: [range, [1,10,2]] +\end{lstlisting} +\paragraph{Note:} The \url{set,run,major}, and \url{minor} values are nested under "Plot-type" + +\bigskip +\textbf{Full example for data selection:} +\begin{lstlisting} + plot1: + timeseries: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit] + var2: [equity] + set: [1] + run: [1,2] + major: [range,[6020,26000,20]] + minor: [1,5,7] # only consider agents 1,5,7 +\end{lstlisting} + +\begin{itemize} +\item \url{summary}: Specify the type of statistical summary. This is also nested under Plot-type. + \begin{itemize} + \item Possible values: \url{full,mean,median,custom_quantile,upper_quartile,lower_quartile,maximum,minimum}. + \item full: the full data ensemble + \item \url{quantile_values}: Specify a quantile range + \begin{itemize} + \item \url{lower_quantile}: \url{[0-1.0]} + \item \url{upper_quantile}: \url{[0-1.0]} + \end{itemize} + \end{itemize} +\end{itemize} + + +\textbf{Example for summary} +To show a simple mean of the data, use: +\begin{lstlisting} + plot1: + timeseries: + summary: mean +\end{lstlisting} + +To show an an interquantile range, use: +\begin{lstlisting} + plot1: + timeseries: + summary: custom_quantile + quantile_values: + lower_quantile: 0.20 + upper_quantile: 0.80 +\end{lstlisting} + + +\bigskip +\textbf{Complete example config.yaml file} + +A complete example \url{config.yaml} file may look like this: +\begin{lstlisting} + i/o: + input_path: '/home/etace/' + + input_files: + Bank: 'Bank.h5' + Eurostat: 'Eurostat.h5' + output_path: '/home/etace/timeseries' + + plot1: + timeseries: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit] + var2: [equity] + set: [1] + run: [1,2] + major: [range,[6020,26000,20]] + minor: [1,5,7] + summary: mean + + plot2: + boxplot: + agent: Eurostat + analysis: multiple_run + variables: + var1: [total_credit] + set: [1] + run: [1] + major: [range,[6020,6900,20]] + minor: [1,8] + summary: custom_quantile + quantile_values: + lower_quantile: 0.20 + upper_quantile: 0.80 +\end{lstlisting} + + +\subsection{File: plot config.yaml} + +For every plot that is specified in the main configuration file \url{config.yaml}, the file \url{plot_config.yaml} provides further specifications for the plot (i.e., the line style, axes labels, legend placement, etc.). The \url{plot_config.yaml} file contains all the necessary configurations for each plot to be generated, and is linked by the \textbf{Plot-key} as specified in the \url{config.yaml} file. Below we explain all settings in the \url{plot_config.yaml} file.\footnote{The plotting options follow the specifications of the \url{matplotlib} library, which is the default library to plot data with Python pandas.} + +\begin{itemize} +\item \textbf{Plot-key} (e.g., \url{plot1}): This string should be the same as the \textbf{Plot-key} in the \url{config.yaml} file, to make sure the correct parameters are mapped to the respective plotting modules. + +\item \url{number_plots}: Specifies how many plots will be output per variable for a particular agent type. +\begin{itemize} +\item Possible values: \url{one,many}. +\item \url{one}: all data series will be displayed in a single plot graph. For time series plots this means: many lines in one plot graph. +\item \url{many}: each data series is displayed in a separate plot graph. There will be as many plot graphs as there are data series selected. +\end{itemize} + +\item \url{plot_name}: Specify a filename for the plot. +(Note: In case of multiple plots, a numerical suffix (in increasing order) is added after the specified file name.) + +\item Plotting limits: +\begin{itemize} +\item \url{l_lim}: y-axis lower limit. +\item \url{u_lim}: y-axis upper limit. +\item \url{tmin}: x-axis lower limit (time axis). +\item \url{tmax}: x-axis upper limit (time axis). +\item Possible values: \url{no, numeric}. (\url{no} means: automatic scaling) +\end{itemize} + +\item \url{plot_legend}: Specify whether a legend for the plot should be displayed. + +\item \url{legend_location}: Specify the location of the legend, either inside or outside of the graph box. +\begin{itemize} +\item Possible values: \url{in,out}. +\end{itemize} + +\item \url{legend_label}: Specify a name for the lines in the plot. Can be any string value. + +\item \url{xaxis_label}: Specify a label for the x-axis. Can be any combination of string values. + +\item \url{yaxis-label}: Specify a label for the y-axis. Can be any combination of string values. + +\item \url{linestyle}: Specify line characteristics. + +\begin{itemize} +\item Possible values: \url{solid,dashed,dashdot,dotted} etc. +\end{itemize} + +\item \url{greyscale}: Specify to plot in greyscale. + +\begin{itemize} +\item Possible values: \url{True, False} (First letter has to be capitalized). +\end{itemize} +\end{itemize} + +\bigskip +\textbf{Complete example plot.yaml file} + +\begin{lstlisting} +plot1: + number_plots: one + plot_name: p1_one_set_multiple_runs_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + greyscale: True + +plot2: + number_plots: one + plot_name: p2_one_set_multiple_runs_ts_quantile.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: red +\end{lstlisting} + + +\bigskip +\textbf{Default settings} +If an option is not specified, then the default settings are: + +\begin{lstlisting} +plot_legend = 'no' +legend_label = None +legend_location = 'best' +plot_type = None +number_plots = 'one' +plot_name = 'default_fig.png' +l_lim = None +u_lim = None +linestyle = 'solid' +marker = 4 +markerfacecolor = None +markersize = None +facecolors = None +plot_title = None +xaxis_label = None +yaxis_label = None +number_bins = 50 +histtype = 'bar' +stacked = False +normed = 1 +fill = False +fillcolor = 'black' +greyscale = False +number_bars = 5 +\end{lstlisting} + +\subsection{File: transform.yaml} + +The \url{transform.yaml} file contains all the specifications for any data transformation. Whenever a transformation is specified in the \url{config.yaml} file, the \url{transform.yaml} file is read to perform the required transformations, and store the resulting data in a new output HDF5 file. + +\bigskip +The parameters in the \url{transform.yaml} file are as follows: + +\begin{itemize} +\item \textbf{Plot-key} (e.g., \url{plot1}): This string should be the same as the \textbf{Plot-key} used in the \url{config.yaml} file for identifying the data transformation block, to make sure the correct parameters are mapped to the respective data transformation modules. +(Note: Although it is called Plot-key, the transform case is an exception and no plots are produced in this case.) + +\item \url{variables}: Variables from the particular agent-type that are to be transformed. The sub-hierarchy \url{var1,var2} etc. allows +the input of multiple variables for any agent type. + +\item \url{transform_function}: The transformation function to apply for the given variables. + +\paragraph{Note:} Other elementary functions such as \textbf{sum}, \textbf{difference}, \textbf{product}, and \textbf{division} can also be performed, which will be added as custom functions in a future release. + +\item \url{aggregate}: If the transformation is to be performed after calculating the summary stats, then a necessary aggregation method can be specified. +\begin{itemize} +\item Possible values: \url{mean,median,maximum,minimum,custom_quantile,upper_quartile,lower_quartile}. +\end{itemize} + +\item \url{write_file}: Specify whether to write the transformation to a file. +\begin{itemize} +\item Possible values: \url{yes,no}. +\end{itemize} + +\item \url{output_path}: If the \url{write_file} option above is set to \url{yes}, then an output path for the file needs to be specified. +This can be any valid file path, as a string, including the filename. + +\item \url{hdf_groupname}: Specify the rootname for the HDF5 group name (internal hierarchy) for the transformed variable. Can be any valid string. +\end{itemize} + +\subsubsection{Definitions of data transformations} +Possible transform functions are listed in Table \ref{Table: Transformation}. + +\begin{table}[hbt!] +\label{Table: Transformation} +\begin{tabular}{|c|c|l|} +\hline \hline +Function & Formula & Description\\ +\hline +\url{mom} & $x_{t}/x_{t-1}$ & \parbox{7cm}{Month-on-month growth rate at monthly frequency. This month wrt. previous month, monthly data.} \\ +\hline +\url{mom_annual} & $x_{t}/x_{t-12}$ & \parbox{7cm}{Monthly growth rate at annual frequency. 12-month trailing sums of monthly observations, growth in month m wrt. same month in previous year.} \\ +\hline +\url{qoq_annual} & $\sum_{s=0}^2 x_{t-s}/\sum_{s=12}^{14} x_{t-s}$ & \parbox{7cm}{Quarterly growth rate at annual frequency. 3-month trailing sums of monthly observations, growth rate in quarter q wrt. same quarter in previous year.} \\ +\hline +\url{pct_change_qoq} & $\sum_{s=1}^{4} x_{t-s}/\sum_{s=4}^{6} x_{t-s}$ & \parbox{7cm}{Quarterly growth rate (quarter-on-quarter). Ratio of two successive 3-month trailing sums of monthly observations.} \\ +\hline +\url{pct_change_yoy} & $\sum_{s=0}^{11} x_{t-s}/\sum_{s=12}^{23} x_{t-s}$ & \parbox{7cm}{Annual growth rate (year-on-year). Ratio of two successive 12-month trailing sums of monthly observations.} \\ +\hline +\end{tabular} +\caption{Transformation functions for commonly used growth rates.} +\end{table} + + +\bigskip +\textbf{Complete transform.yaml file} + +A particular complete \url{transform.yaml} file may look as follows: +\begin{lstlisting} + plot2: + variables: + var1: [total_credit] + var2: [equity] + transform_function: m_o_m_a + aggregate: mean + + write_file: yes + output_path: '/home/etace/transformed.h5' + hdf_groupname: 'growth_rates' +\end{lstlisting} + + +\clearpage +\section{Walkthrough and Tutorial examples} + + +In this section we give examples of input configuration files and their respective output plots. +Once the necessary parameters are set, by following the instructions specified in Section \ref{Section:configuration}, the module can be run to get the desired results. To demonstrate some of the functionalities, the parameters of the configuration files are shown below, along with the plots they yield. + +Using the dataset + + \url{https://github.com/svdhoog/FLAViz/tree/master/data/visualisation}, + + and the following parameter settings, the following plots can be produced. + +\clearpage +\subsubsection{Example 1} +For agent Firm, one set, ten runs, 80 agent instances, plotted in a single plot. + +\begin{lstlisting} +config.yaml:: + + plot1: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [13] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot.yaml:: + + plot1: + number_plots: one + plot_name: one_set_multiple_runs_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.7]{./plots_tut_2/one_set_multiple_runs_timeseries_price.png} +\caption{\footnotesize Example 1: one set, multiple runs, timeseries, price.} +\end{figure} + +\clearpage +\subsubsection{Example 2} +For agent Firm, one set, one run, 20-80 quantiles of population distribution across 80 agent instances, plotted in a single plot. + +\begin{lstlisting} +config.yaml:: + + plot2: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +plot.yaml:: + + plot2: + number_plots: one + plot_name: one_set_multiple_runs_ts_quantile.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: red +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.6]{./plots_tut_2/one_set_multiple_runs_ts_quantile_0.png} +\caption{\footnotesize Example 2: Quantiles 20 and 80 of the population distribution across 80 agent instances.} +\end{figure} + +\clearpage +\subsubsection{Example 3} + For agent Firm, one set, one run, 80 agent instances, boxplot. + +\begin{lstlisting} +config.yaml:: + + plot3: + boxplot: + agent: Firm + analysis: multiple_set + variables: + var1: [price] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + +plot.yaml:: + + plot3: + number_plots: one + plot_name: one_set_one_run_bp_price.png + plot_legend: yes + legend_label: (Agent = Firm, var = Price) + legend_location: best + xaxis_label: Time + yaxis_label: Distribution over price + number_bars: 5 +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.7]{./plots_tut_2/one_set_one_run_bp_price_price.png} +\caption{\footnotesize Example 3: boxplots of the population distribution across 80 agent instances, the number of boxplots can be specified (here: 5).} +\end{figure} + +\clearpage +\subsubsection{Example 4} +For agent Firm, one set, 20 runs, 80 agent instances, scatter plot of two variables. + +\begin{lstlisting} +config.yaml:: + + plot4: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + var2: [output] + delay: no + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot.yaml:: + + plot4: + number_plots: one + plot_name: one_set_multiple_runs_sp_price_output.png + plot_legend: yes + legend_location: best + legend_label: price vs. output + linestyle: solid + marker: + +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.7]{./plots_tut_2/one_set_multiple_runs_sp_price_output_0.png} +\caption{\footnotesize Example 4: Scatter plot of 2 variables, for a data set consisting of 20 runs, 80 agent instances. We first took the mean across the agents, then plot the ensemble data across all runs, and all iterations (the plot shows $20\times 325$ points).} +\end{figure} + +\clearpage +\subsubsection{Example 5} + For agent Firm, one set, 20 runs each, 80 agent instances, delay plot for one variable. + +\begin{lstlisting} +config.yaml:: + + plot5: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + delay: yes + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot.yaml:: + + plot5: + number_plots: one + plot_name: one_set_multiple_runs_sp_price_delay.png + plot_legend: yes + legend_location: best + legend_label: price vs. price_delay + linestyle: solid + marker: + +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.7]{./plots_tut_2/one_set_multiple_runs_sp_price_delay_0.png} +\caption{\footnotesize Example 5: Delay plot $(x_t, x_{t+1})$.} +\end{figure} + +\clearpage +\subsubsection{Example 6} +For agent Firm, one set, one run, 80 agent instances, histogram of one variable. + +\begin{lstlisting} +config.yaml:: + + plot6: + histogram: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot.yaml:: + + plot6: + number_plots: one + plot_name: one_set_one_run_hg_price.png + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill: yes + stacked: False + legend_location: best + xaxis_label: xlabel + yaxis_label: ylabel +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.7]{./plots_tut_2/one_set_one_run_hg_price_price.png} +\caption{\footnotesize Example 6: Histogram of a single variable across the agent population.} +\end{figure} + +\clearpage +\subsubsection{Example 7} + For agent Firm, one set, 20 runs, 80 agent instances, histogram of the ensemble distribution over all sets. + +\begin{lstlisting} +config.yaml:: + + plot7: + histogram: + agent: Firm + analysis: multiple_set + variables: + var1: [price] + set: [10] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot.yaml:: + + plot7: + number_plots: one + plot_name: one_set_multiple_runs_hg_price.png + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill: no + stacked: False + legend_location: best + xaxis_label: xlabel + yaxis_label: ylabel +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.6]{./plots_tut_2/one_set_multiple_runs_hg_price_price.png} +\caption{\footnotesize Example 7: Histogram of a single variable across 20 runs, and across the agent population. Shown is the ensemble distribution across all runs, by concatenating and flattening the data across all dimensions.} +\end{figure} + + +\clearpage +\subsubsection{Example 8} +For agent Firm, four sets, 20 runs each, 80 agent instances, the variable price is plotted in a single plot. + +\begin{lstlisting} +config.yaml:: + + plot8: + timeseries: + agent: Firm + analysis: multiple_batch + variables: + var1: [output] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot.yaml:: + + plot8: + number_plots: one + plot_name: timeseries_agentanalysis.png + plot_legend: yes + legend_location: best + x-axis label: Time + y-axis label: output + linestyle: dashed + marker: None +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.6]{./plots_tut_2/P4_ts_price.png} +\caption{\footnotesize Example 8: Time series plot of multiple sets (4), multiple runs (20), and multiple agent instances (80). Each line represents one set, and displays the mean across runs, and across agents.} +\end{figure} + + + +\clearpage +\subsubsection{Example 9} +For agent Firm, 4 sets, 20 runs each, 80 agent instances, 20-80 quantiles of the population distribution of the variable price are plotted in a single plot. + +\begin{lstlisting} +config.yaml:: + + plot7: + timeseries: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + + +plot.yaml:: + + plot7: + number_plots: one + plot_name: ts_multibatch_analysis.png + plot_legend: yes + legend_location: best + x-axis label: Time + y-axis label: price + linestyle: solid + marker: None + fill_between: yes +\end{lstlisting} + +\begin{figure}[b!]\centering\leavevmode +\includegraphics[scale=.6]{./plots_tut_2/P7_ts_price.png} +\caption{\footnotesize Example 9: Quantile ranges of the variable 'price' for multiple sets (4), multiple runs (20), and multiple agent instances (80). For each set the quantile range (20-80) is displayed.} +\end{figure} + + +\section{FLAViz FAQ} + +\begin{itemize} +\item[Q:] Why is FLAViz based on Python pandas and matplotlib? + +A: Python pandas is open source, has an active user community, and has many developers. It is the current library of choice for time series data analysis. +It provides many build-in statistical functionalities. +There are two main functionalities of pandas that are especially important for us: + +\begin{itemize} +\item hierarchical indexing: this allows us to use a high dimensional data frame (the ndarray format). +\item bygroup: this allows us to rotate the hierarchical index. +\end{itemize} + +\item[Q:] Why the conversion from XML files to HDF5 format? + +A: The XML files that FLAME outputs is a fully tagged data format, so it is very verbose. For large scale simulations this is prohibitive, due to the sheer size of the data volumes this generates. +To reduce this storage footprint, but still keep all data together in a structured format, the HDF5 standard was chosen for its hierarchical structure. +The data for each agent type is stored in a single HDF5 file, which can be of any size (we have so far dealt with single files of up to 20 GB without any problems). +Inside of the HDF5 file there is a POSIX-style folder hierarchy, with data groups and data sets. A particular requirement for HDF5 is that the 'data set' has a homogeneous data structure. +For this data structure we have selected the 3D DataFrame from Python pandas (but we should note that the current development of Python pandas goes soo quick that currently this is shifting to the ndarray, which will replace the 3D Data Panel in a more generic data format). + +\item[Q:] How do you do the data transformations, data selection and data filtering? + +We use the bygroup function of pandas to sort the hierarchical index, and to rotate the data frame. + +\item[Q:] Can I use FLAViz with other agent-based simulation platforms than FLAME? + +Yes, you can! The only requirement is that the final data output is either in XML or in HDF5 format (see the file format specifications). +\end{itemize} + +\end{document} diff --git a/docs/manual/tex/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files.png b/docs/manual/tex/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files.png new file mode 100644 index 0000000..02d836e Binary files /dev/null and b/docs/manual/tex/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files.png differ diff --git a/docs/manual/tex/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files_with_file_size.png b/docs/manual/tex/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files_with_file_size.png new file mode 100644 index 0000000..c9367ef Binary files /dev/null and b/docs/manual/tex/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files_with_file_size.png differ diff --git a/docs/manual/tex/hdf5_snapshots/Fig_1_individual_hdf5_corresponding_to_db_agent_1.png b/docs/manual/tex/hdf5_snapshots/Fig_1_individual_hdf5_corresponding_to_db_agent_1.png new file mode 100644 index 0000000..4e8fa45 Binary files /dev/null and b/docs/manual/tex/hdf5_snapshots/Fig_1_individual_hdf5_corresponding_to_db_agent_1.png differ diff --git a/docs/manual/tex/hdf5_snapshots/Fig_2_individual_hdf5_corresponding_to_db_agent_2.png b/docs/manual/tex/hdf5_snapshots/Fig_2_individual_hdf5_corresponding_to_db_agent_2.png new file mode 100644 index 0000000..f0f669f Binary files /dev/null and b/docs/manual/tex/hdf5_snapshots/Fig_2_individual_hdf5_corresponding_to_db_agent_2.png differ diff --git a/docs/manual/tex/hdf5_snapshots/Fig_3_hdf5_file_per_agent_showing_values.png b/docs/manual/tex/hdf5_snapshots/Fig_3_hdf5_file_per_agent_showing_values.png new file mode 100644 index 0000000..d028d6c Binary files /dev/null and b/docs/manual/tex/hdf5_snapshots/Fig_3_hdf5_file_per_agent_showing_values.png differ diff --git a/docs/manual/tex/hdf5_snapshots/Fig_4_list_of_hdf5_files_separated-per-agent-type.png b/docs/manual/tex/hdf5_snapshots/Fig_4_list_of_hdf5_files_separated-per-agent-type.png new file mode 100644 index 0000000..dcaf613 Binary files /dev/null and b/docs/manual/tex/hdf5_snapshots/Fig_4_list_of_hdf5_files_separated-per-agent-type.png differ diff --git a/TODO/doc/images/P1_ts_technology.png b/docs/manual/tex/images_tut_1/P1_ts_technology.png similarity index 100% rename from TODO/doc/images/P1_ts_technology.png rename to docs/manual/tex/images_tut_1/P1_ts_technology.png diff --git a/TODO/doc/images/P2_ts_total_debt.png b/docs/manual/tex/images_tut_1/P2_ts_total_debt.png similarity index 100% rename from TODO/doc/images/P2_ts_total_debt.png rename to docs/manual/tex/images_tut_1/P2_ts_total_debt.png diff --git a/TODO/doc/images/P3_ts_output.png b/docs/manual/tex/images_tut_1/P3_ts_output.png similarity index 100% rename from TODO/doc/images/P3_ts_output.png rename to docs/manual/tex/images_tut_1/P3_ts_output.png diff --git a/TODO/doc/images/P4_ts_price.png b/docs/manual/tex/images_tut_1/P4_ts_price.png similarity index 100% rename from TODO/doc/images/P4_ts_price.png rename to docs/manual/tex/images_tut_1/P4_ts_price.png diff --git a/TODO/doc/images/P5_ts_unemployment_rate.png b/docs/manual/tex/images_tut_1/P5_ts_unemployment_rate.png similarity index 100% rename from TODO/doc/images/P5_ts_unemployment_rate.png rename to docs/manual/tex/images_tut_1/P5_ts_unemployment_rate.png diff --git a/TODO/doc/images/P6_ts_monthly_output.png b/docs/manual/tex/images_tut_1/P6_ts_monthly_output.png similarity index 100% rename from TODO/doc/images/P6_ts_monthly_output.png rename to docs/manual/tex/images_tut_1/P6_ts_monthly_output.png diff --git a/TODO/doc/images/P7_ts_price.png b/docs/manual/tex/images_tut_1/P7_ts_price.png similarity index 100% rename from TODO/doc/images/P7_ts_price.png rename to docs/manual/tex/images_tut_1/P7_ts_price.png diff --git a/docs/manual/tex/plots_tut_2/P4_ts_price.png b/docs/manual/tex/plots_tut_2/P4_ts_price.png new file mode 100644 index 0000000..bba7577 Binary files /dev/null and b/docs/manual/tex/plots_tut_2/P4_ts_price.png differ diff --git a/docs/manual/tex/plots_tut_2/P7_ts_price.png b/docs/manual/tex/plots_tut_2/P7_ts_price.png new file mode 100644 index 0000000..e592e4e Binary files /dev/null and b/docs/manual/tex/plots_tut_2/P7_ts_price.png differ diff --git a/TODO/doc/plots/one_set_multiple_runs_hg_price_price.png b/docs/manual/tex/plots_tut_2/one_set_multiple_runs_hg_price_price.png similarity index 100% rename from TODO/doc/plots/one_set_multiple_runs_hg_price_price.png rename to docs/manual/tex/plots_tut_2/one_set_multiple_runs_hg_price_price.png diff --git a/docs/manual/tex/plots_tut_2/one_set_multiple_runs_sp_price_delay_0.png b/docs/manual/tex/plots_tut_2/one_set_multiple_runs_sp_price_delay_0.png new file mode 100644 index 0000000..20a0c42 Binary files /dev/null and b/docs/manual/tex/plots_tut_2/one_set_multiple_runs_sp_price_delay_0.png differ diff --git a/TODO/doc/plots/one_set_multiple_runs_sp_price_output_0.png b/docs/manual/tex/plots_tut_2/one_set_multiple_runs_sp_price_output_0.png similarity index 100% rename from TODO/doc/plots/one_set_multiple_runs_sp_price_output_0.png rename to docs/manual/tex/plots_tut_2/one_set_multiple_runs_sp_price_output_0.png diff --git a/TODO/doc/plots/one_set_multiple_runs_timeseries_price.png b/docs/manual/tex/plots_tut_2/one_set_multiple_runs_timeseries_price.png similarity index 100% rename from TODO/doc/plots/one_set_multiple_runs_timeseries_price.png rename to docs/manual/tex/plots_tut_2/one_set_multiple_runs_timeseries_price.png diff --git a/TODO/doc/plots/one_set_multiple_runs_ts_quantile_0.png b/docs/manual/tex/plots_tut_2/one_set_multiple_runs_ts_quantile_0.png similarity index 100% rename from TODO/doc/plots/one_set_multiple_runs_ts_quantile_0.png rename to docs/manual/tex/plots_tut_2/one_set_multiple_runs_ts_quantile_0.png diff --git a/TODO/doc/plots/one_set_one_run_bp_price_price.png b/docs/manual/tex/plots_tut_2/one_set_one_run_bp_price_price.png similarity index 100% rename from TODO/doc/plots/one_set_one_run_bp_price_price.png rename to docs/manual/tex/plots_tut_2/one_set_one_run_bp_price_price.png diff --git a/TODO/doc/plots/one_set_one_run_hg_price_price.png b/docs/manual/tex/plots_tut_2/one_set_one_run_hg_price_price.png similarity index 100% rename from TODO/doc/plots/one_set_one_run_hg_price_price.png rename to docs/manual/tex/plots_tut_2/one_set_one_run_hg_price_price.png diff --git a/docs/manual/tex/png/data_processing.pdf b/docs/manual/tex/png/data_processing.pdf new file mode 100644 index 0000000..70c87a6 Binary files /dev/null and b/docs/manual/tex/png/data_processing.pdf differ diff --git a/docs/manual/tex/png/data_processing.png b/docs/manual/tex/png/data_processing.png new file mode 100644 index 0000000..a66002c Binary files /dev/null and b/docs/manual/tex/png/data_processing.png differ diff --git a/docs/manual/tex/png/data_processing.xcf b/docs/manual/tex/png/data_processing.xcf new file mode 100644 index 0000000..007c40e Binary files /dev/null and b/docs/manual/tex/png/data_processing.xcf differ diff --git a/docs/manual/tex/png/flavis_package_diagram.pdf b/docs/manual/tex/png/flavis_package_diagram.pdf new file mode 100644 index 0000000..5bb77bb Binary files /dev/null and b/docs/manual/tex/png/flavis_package_diagram.pdf differ diff --git a/docs/manual/tex/png/simulation_workflow.pdf b/docs/manual/tex/png/simulation_workflow.pdf new file mode 100644 index 0000000..29a073b Binary files /dev/null and b/docs/manual/tex/png/simulation_workflow.pdf differ diff --git a/docs/manual/tex/png/simulation_workflow.png b/docs/manual/tex/png/simulation_workflow.png new file mode 100644 index 0000000..3b30ece Binary files /dev/null and b/docs/manual/tex/png/simulation_workflow.png differ diff --git a/docs/manual/tex/png/simulation_workflow.xcf b/docs/manual/tex/png/simulation_workflow.xcf new file mode 100644 index 0000000..453fd23 Binary files /dev/null and b/docs/manual/tex/png/simulation_workflow.xcf differ diff --git a/docs/replace_xml_writeout_functions/readme.md b/docs/replace_xml_writeout_functions/readme.md new file mode 100644 index 0000000..7c830a2 --- /dev/null +++ b/docs/replace_xml_writeout_functions/readme.md @@ -0,0 +1,50 @@ +Replace write_out functions in file xml.c +==== +This replacement script is useful for reducing the storage footprint of FLAME simulations. It allows the user to create a "shadow model.xml" file with a subset of the agent memory variables, resulting in fewer variables than the full model.xml file. The script replace_xml_functions.py takes as input the full-model xml.c file and the shadow-model xml.c file, and then replaces all functions called `write_agentname_Agent()` in the full-model xml.c file. The resulting full-model xml.c file can be compiled as usual with rest of the FLAME model C code. The resulting simulation will only output the subset of variables for each agent type. + +- src/replace_xml_functions.py: Python script for replacing XML writeout functions in xml.c (FLAME-models) +- Usage: + + python replace_xml_functions.py xml.c shadow_xml.c path_to_shadow_model shadow_model.xml + + where: + - xml.c: original xml.c file from complete model + - shadow_xml.c: xml.c file from 'shadow' model, with a smaller set of variables (the xml.c file results from xparsing the shadow model.xml file) + - path_to_shadow_model: full path to shadow model + - shadow_model.xml: model.xml file containing all agent types, but fewer variables + +Test examples +==== +These test examples are for testing the functionality of the script replace_xml_writeout_functions.py. + +How to run: + python replace_xml_writeout_functions.py [xml.c file] [shadow_xml.c file] [SHADOW_MODEL folder] [shadow_model.xml file] [-v] + +Arguments: + +- file1 = Name of original xml.c file, containing full code for the functions writeout_Agentname_agent() +- file2 = Name of shadow_xml.c file, containing replacement code for the functions writeout_Agentname_agent() +- modelpath = Path to the folder of shadow model xml +- shadow model file = Name of the shadow model xml file + +Options: + + -v, --verbose = Get the status of the intermediate processing steps + +Example run commands: + +- If shadow model is in the current folder, and the script is also run from the current folder: + + python replace_xml_writeout_functions.py xml.c shadow_xml.c . shadow_model.xml -v >out.txt + +- If the files are in separate input folders: + + xml.c: inputs/REAL_MODEL + shadow_xml.c: inputs/XML + shadow_model.xml: inputs/SHADOW_MODEL, + +then use: + + python replace_xml_writeout_functions.py inputs/REAL_MODEL/xml.c inputs/XML/shadow_xml.c inputs/SHADOW_MODEL shadow_model.xml -v >out.txt + +Output files are in output/. diff --git a/docs/slides/Slides-Conquaire-Etace.tex b/docs/slides/Slides-Conquaire-Etace.tex new file mode 100644 index 0000000..a4fa092 --- /dev/null +++ b/docs/slides/Slides-Conquaire-Etace.tex @@ -0,0 +1,971 @@ +%PDFLatex +\documentclass[9pt,sansserif]{beamer} + +%Latex->DVI->PS->PDF +%\documentclass[9pt,sansserif,blue,dvips,ignorenonframetext]{beamer} + +%Notes: +%\documentclass[9pt,handout,notes=show]{beamer} + +%Notes only: only produces .div, so it needs previous compilation of slides mode +%\documentclass[9pt,handout,notes=only]{beamer} + +%The following sets notes to use no header +\setbeamertemplate{note page}{% + \insertnote% +} + +\mode +{ +\usepackage{beamerthemesplit} + +\newlength{\parskipbackup} +\setlength{\parskipbackup}{\parskip} +\newlength{\parindentbackup} +\setlength{\parindentbackup}{\parindent} + +\let\notebackup\note +\renewcommand{\note}[1]{\notebackup{% + \mode{\addtocounter{page}{-1}}% + \setlength{\parindent}{0ex}% + \setlength{\parskip}{10pt}% + \noindent% + {\normalsize{}#1}% + \setlength{\parskip}{\parskipbackup}% + \setlength{\parindent}{\parindentbackup}% +}% +} +} + +\mode +{ + +%\usepackage{beamerthemesplit} +%\usepackage{beamerthemebars} +\usepackage{beamerthemelined} +%\usepackage{beamerthemetree} +%\usepackage{beamerthemetreebars} + + + %1st CHOICE: + %\usetheme{Darmstadt} + %2nd CHOICE: + %\usetheme{Dresden} + %3rd CHOICE: + %\usetheme{Warsaw} + %\setbeamercovered{transparent} + %4th CHOICE: + %\usetheme{Marburg} + +%\usetheme{Copenhagen} +%\usetheme{Darmstadt} +%\usetheme{Dresden} +%\usetheme{Frankfurt} +%\usetheme{Goettingen} +% +%\usetheme{Madrid} %lots of free space +%\usetheme{Malmoe} %much free space +% +%\usetheme{Marburg} %Outline right panel +%\usetheme{Montpellier} +%\usetheme{PaloAlto} %Outline left panel +%\usetheme{Pittsburgh} +%\usetheme{Singapore} +%\usetheme{Warsaw} +} + +\mode
+{ + %\usepackage{fullpage} + \usepackage{pgf} + \usepackage{hyperref} + \setjobnamebeamerversion{beamerexample2.beamer} +} + +\usepackage[english]{babel} +\usepackage[latin1]{inputenc} + +\usepackage{mathptmx} +\usepackage[T1]{fontenc} + +\usepackage{times} +\usepackage{amsfonts} +\usepackage{amssymb} +\usepackage{amsmath} +\usepackage{euscript} + +\usepackage{color} +\usepackage{fancybox} +\usepackage{graphicx} +\usepackage{boxedminipage} +\usepackage{lscape} +\usepackage{epsfig,psfrag,graphicx} +\usepackage{url, verbatim} +\usepackage{epsfig, graphicx} +\usepackage{natbib} +\usepackage{booktabs, longtable, listings} + + +%% Define a new 'code' style for the url package that will use a sans serif font. +\makeatletter +\def\url@codestyle{% + \@ifundefined{selectfont}{\def\UrlFont{\sf}}{\def\UrlFont{\sffamily}}} +\makeatother +%% Now actually use the newly defined style. +\urlstyle{code} + + +% ------------------------------------------------------------------------ +% PREAMBLE 2 - Slide Background Format +% ------------------------------------------------------------------------ +\newcommand{\sref}[1]{SLIDE \ref{#1}} + +\definecolor{darkred}{RGB}{130,0,0} +\definecolor{darkgreen}{RGB}{0,130,0} +\definecolor{darkblue}{RGB}{0,0,130} + +\newcommand{\RR}[1]{{\color{colorSG5} #1}} +\newcommand{\GG}[1]{{\color{green} #1}} +%\newcommand{\BB}[1]{{\color{blue} #1}} +\newcommand{\YY}[1]{{\color{yellow} #1}} +\newcommand{\CC}[1]{{\color{cyan} #1}} +\newcommand{\VV}[1]{{\color{violet} #1}} + +%\newcommand{\BA}[1]{{\color{colorSG} #1}} +\newcommand{\BB}[1]{{\color{colorSG1} #1}} +\newcommand{\BC}[1]{{\color{colorSG2} #1}} +\newcommand{\GO}[1]{{\color{colorSG4} #1}} + +\newcommand{\DR}[1]{{\color{darkred} #1}} +\newcommand{\DG}[1]{{\color{darkgreen} #1}} +\newcommand{\DB}[1]{{\color{darkblue} #1}} + +\newcommand{\ce}[1]{ \vspace{-0.7cm} \begin{center} #1 \end{center} \vspace{-0.35cm} } + +% CHANGED: different definition of \heading +\newcommand{\heading}[1]{\begin{center}\large\bf #1\end{center}} +\let\heading=\frametitle + +% ------------------------------------------------------------------------ + +% ------------------------------------------------------------------------ +% DOCUMENT +% ------------------------------------------------------------------------ +% The following info should normally be given in your main file: +% [] indicates the text that is put on the bottom of each slide, remove when same as main titlepage text. + +\begin{document} + + +\title[Agent-Based Data Analysis with Python pandas]{Agent-Based Data Analysis with Python pandas\\\bigskip FLAViz: Flexible Large-scale Agent Visualization Library} +\author[Krishna Devkota]{by Krishna Devkota\\\& Sander van der Hoog} +\date{24 Nov 2017} + +\setcounter{tocdepth}{1} + + + +%\section{Outline} : entry in top bar +%\subsection{Outline}: entry in second bar and open dots for slides +%\frametitle{Outline}: produces a large title heading in main text field +\frame{ + +\begin{figure}[hbt] +\includegraphics[scale=.6]{./logos/unibi-logo-gruen.png} +\hspace{.1cm} +\includegraphics[scale=.02]{./logos/ETACE_new.png} +\hspace{.1cm} +\includegraphics[scale=.6]{./logos/dfg_logo_blau.jpg} +\end{figure} + +\titlepage} +\note{ +\small +This talk is about: +\begin{itemize} +\item A progress report on the Conquaire project, which is a DFG project on research data management and quality assurance +\item Our focus here is on economic data analysis, in particular on agent-based models and data from economic computer simulation models. +\end{itemize} +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Our point of departure} + +\bigskip +\textbf{Continuous quality control for research data to ensure analytical reproducibility} + +\bigskip +Conditions for analytical reproducibility: +\begin{itemize} +\item the primary or secondary data are available for inspection and processing +\item the data is syntactically well-formed and follows best practices from the corresponding community +\item the analytic procedures (e. g. scripts, spreadsheets, etc.) that were used to process or analyse the data are available +\item the analytic procedures can be run on the data to reproduce the actual result published in a paper. +\end{itemize} + + +\begin{figure}[hb!] +\centering\leavevmode +\includegraphics[scale=.75]{./png/conquaire_kick-off-meeting-slides.pdf} +\end{figure} + +\end{frame} +\note{ +Conquaire project: +\begin{itemize} +\item Conquaire is a DFG project at Bielefeld University on research data management and quality assurance +\item The focus is on analytical reproducibility, referring to the (computational) analysis of scientific data (can be primary or secondary data) +\end{itemize} + +Conditions for analytical reproducibility: +\begin{itemize} +\item the primary or secondary data are available for inspection and processing +\item the data is syntactically well-formed and follows best practices from the corresponding community +\item the analytic procedures (e. g. scripts, spreadsheets, etc.) that were used to process or analyse the data are available +\item and: these analytic procedures can be run on the data to reproduce the actual result published in a paper. +\end{itemize} +} + +%-------------------------------------------------------------------------- + +\begin{frame}{Agent-Based Data Analysis with pandas}\small +%\vspace{-1cm} +%\thispagestyle{empty} +%\textbf{Two ingredients central to our work flow:} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{../png/}} +% +\begin{minipage}{16.5cm} +%\hspace{-1cm} +%\begin{boxedminipage}{6cm} +\begin{minipage}{6cm} +\centering\leavevmode +{\bf FLAME:\\Simulation \& Data Generation} +\vspace{.5cm}\\ +\includegraphics[scale=.3]{./png/flame.png}\\ +\onslide<2> {\vspace{.8cm}\includegraphics[scale=.25]{./png/python-pandas-screenshot.png}} +\end{minipage} +%\end{boxedminipage} +% +%\begin{boxedminipage}{5cm} +\begin{minipage}{5cm} +\centering\leavevmode +{\bf pandas:\\Data Munging \& Processing} +\vspace{.5cm}\\ +\includegraphics[scale=.5]{./png/panda-eat.jpg}\\ +\end{minipage} +%\end{boxedminipage} +% +\end{minipage} +\end{figure} +\onslide<2> {\vspace{.3cm} \textbf{pandas} = Python library for scientific computing using \textbf{Pan}el \textbf{Da}ta \textbf{S}ets.\\ +\textbf{Data munging} = "the process of changing data into another format so that it can be used or processed". +} +\end{frame} +\note{ +Two ingredients central to our work flow: + +- FLAME:\\Simulation \& Data Generation + +- pandas: Data Munging \& Processing +} + +%-------------------------------------------------------------------------- +\section{Intro} +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{What we do: Conquaire @ ETACE} + +%\large +\textbf{Our primary objectives:} + +\bigskip +\begin{enumerate}\itemsep2em + +\item Develop a flexible and modular software library for analyzing data from ABM simulations + +\item Build on pandas: a generic, general-purpose data analysis toolbox in Python + +\item Toolbox components: data munging, data transformation, and data visualization + +\item Create a Data Analysis Toolbox for Agent-based Simulations (DATAS) + +\item Integrate the toolbox into an automated work-flow for research data management (primary Conquaire project mission) + +\end{enumerate} +\end{frame} +\note{ +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Some plumbing of our simulation infrastructure} +%\vspace{-.6cm} +%\thispagestyle{empty} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/}} +% +%\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.175]{flame_pandas_workflow_diagram.pdf} +\end{minipage} +\end{figure} + +\end{frame} +\note{} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Selected topics from big data analysis relevant to our purpose} + +\begin{enumerate}\itemsep2em + +\item Data generation + +\item Data storage + +\item Data selection + +\item Data filtering + +\item Data transformation + +\item Data visualization + +\end{enumerate} +\end{frame} +\note{ +Selected topics from data analytics + +Each of these pose problems when confronted with large data volumes. + +A concerted effort is required to tackle these issues. +} + +%-------------------------------------------------------------------------- +\section{Data generation} +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data Generation, Conversion \& Storage} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/}} +% +\hspace{-2cm} +\begin{minipage}{10cm} +%\centering\leavevmode +%\includegraphics[scale=1]{workflow_cheops.jpg}\\ [.5cm] +\includegraphics[scale=1]{workflow_simulation.jpg}\\ +{Munging + Storage + Processing:}\\ +\includegraphics[scale=0.25]{workflow_data_conversion/workflow_data_conversion.pdf} +\end{minipage} +\end{figure} +\end{frame} +\note{} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data Volume} + +\begin{minipage}{12cm} +% +\begin{minipage}{8cm} +\hspace{-2cm} +\includegraphics[scale=0.5]{./png/set_run_hierarchy.png} +\end{minipage} +% +\begin{minipage}{4cm} +{\onslide<2,3> \includegraphics[scale=0.4]{./png/rubiks-mirror-cube-regular.jpg}} +% +{\onslide<3> \includegraphics[scale=0.4]{./png/rubiks-mirror-cube-mixed-up.jpg}} +\end{minipage} +% +\end{minipage} + +\end{frame} +\note{ +\small +A particular use-case of the ABM simulations is to generate data along 3 dimensions: +\begin{itemize} +\item 513 folders for different parameter constellations +\item 1,000 runs per case +\item 1,000 XML files are generated per run +\end{itemize} + +1st Rubik's Mirror Cube: + +Since we have 3 dimensions of data, the data volume looks a bit like the 3D Cube as shown here. +But since the data sizes differ per run and per parameter setting, the small data blocks inside the 3D cube are not homogeneous in size. +The ideal case would be to have it all nicely ordered as structured data like the picture. + +2nd Rubik's Mirror Cube: + +But the real situation is more like this. The actual data is not nicely ordered, and takes up a lot of space on hard disk because it is all in separate files. +It is also not very easy to deal with when you have to read-in large volumes of data to manipulate it, or to plot something. + +Typical data sizes of the various types of files are: +\begin{itemize} +\item XML files: $513x10^6$ (tiny sand grains, KB) +\item SQL files: $513x10^3$ (small beach pebbles, MB) +\item HDF5 files: 10 (big chunks of data, GB) +\end{itemize} +} + +%-------------------------------------------------------------------------- +\section{Data storage} +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data storage: Our generic use-case for ABM simulations} + +Agent-based data sets vary along 6 dimensions: + +\bigskip +\begin{enumerate}\itemsep2em + +\item Sets: $s=1,...,S$ - Parameter settings (model calibrations) + +\item Runs: $r=1,...,R$ - Monte Carlo replication runs (random seeds) + +\item Iterations: $t=1,..,T$ - Time periods + +\item Agent types: $a=1,...,A$ - Classes, groups of agent sub-populations + +\item Agent instances: $i=1,...,n_a$ - Individual agents ($n_a$= no. agents of type $a$) + +\item Variables: $j=1,...,m$ - Scalars, Arrays, Composites (data structures) +\end{enumerate} + +\end{frame} +\note{ +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data storage (cont.)} + +\vspace{1cm} +A single data point (assuming a scalar variable): + +{\Large +\begin{equation*} +X \in \mathbb{R}^6 \text{ or } +X_{s,r,t,a,i,j} \in \mathbb{R} +\end{equation*}} + +\bigskip +\begin{itemize}\itemsep1em +\item Requirement: keep all the data together, and in some order. + +\item What type of structured data set can deal with this?\\ [5pt] + +\begin{enumerate}\itemsep2em + +\item \onslide<2,3,4,5> R: 2D Dataframe $\rightarrow$ not enough! $\rightarrow$ need folder hierarchy for other 4-Dims? + +\item \onslide<3,4,5> pandas: 3D Panel $\rightarrow$ not enough! $\rightarrow$ need folder hierarchy for other 3-Dims? + +\item \onslide<4,5> HDF5: Hierarchical data format v5 $\rightarrow$ could this be enough? +\begin{itemize} +\item \onslide<5> Data group / data subgroup / data set (POSIX standard) +\end{itemize} + +\end{enumerate} +\end{itemize} + +\end{frame} +\note{ +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data storage: Revised data hierarchy} + +\begin{enumerate}\itemsep1em + +\item {\color{darkgreen}Agent types: $a=1,...,A$} - Classes, groups of agent sub-populations + +\item \hspace{1cm}{\color{darkblue}Sets: $s=1,...,S$} - Parameter settings (model calibrations) + +\item \hspace{1cm}{\color{darkblue}Runs: $r=1,...,R$} - Monte Carlo replication runs (random seeds) + +\item \hspace{2cm}{\color{darkred}Iterations: $t=1,..,T$} - Time periods + +\item \hspace{2cm}{\color{darkred}Agents: $i=1,...,n_a$} - Individual agents (per type) + +\item \hspace{2cm}{\color{darkred}Variables: $j=1,...,m$} - Scalars, Arrays, Composites +\end{enumerate} + +\bigskip +{\color{darkgreen}HDF5 file}: 1 file per agent type: Eurostat, Firm, Household, Bank, ... + +\bigskip +{\color{darkblue}Data groups inside a HDF5 file}: +\begin{itemize} +\item flat form: \url{set_1_run_1}, \url{set_1_run_2}, ... +\item data hierarchy: \url{set_1/run_1}, \url{set_1/run_2}, ... +\end{itemize} + +\bigskip +{\color{darkred}pandas 3D Panel}: homogeneous data block inside each data group + +\end{frame} +\note{ +} + +%-------------------------------------------------------------------------- +\section{Data processing} +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data processing: HDF5 (HDD) $\rightarrow$ DataFrame in pandas (RAM)} + +\bigskip +\begin{enumerate}\itemsep2em + +\item Python pandas: + \begin{itemize}\itemsep1em + \item read-in HDF5 file per agent type into a DataFrame in RAM (parallelizable!) + \item create a flattened DataFrame with hierarchical-index for each agent type + \item 6D data: 5-dim hierarchical index for other 5 dimensions + \item select type of analysis (agent-level, run-level, set-level) + \item use groupby, filter, transform to manipulate data in RAM + \item create summary statistics + \end{itemize} + +\item Store data transformations in temporary DataFrame in RAM + +\item Create visualizations: time series, box plots, histograms, scatter plots + +\item Write out final data to disk: hdf5, png, pdf, csv + +\end{enumerate} +\end{frame} +\note{ +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +%\frametitle{} +\vspace{-1cm} +\thispagestyle{empty} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.45]{flame_pandas_workflow_diagram.pdf} +\end{minipage} +\end{figure} + +\end{frame} +\note{ +\textbf{Flame with pandas workflow diagram} + +\underline{Module Blue:} Data input module + +a. Read-in from HDF5, loop across agent types + +\underline{Module Green:} Data preparation module + +b. In RAM, create a data frame for each agent type to hold all agent instances of that type + +\underline{Module Red:} Data selection module + +c. Data selection: filter the data according to a list of variables for each agent type + +d. Data filtering: filter the data according to filter criteria (X>0, ID==2,...) + +\underline{Module Green:} Data aggregation module + +e. Data aggregation: create summary statistics of the data + +f. Pass filtered data frame with summary statistics (in RAM) to the next modules +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +%\frametitle{} +\vspace{-7.7cm} +\thispagestyle{empty} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.40]{flame_pandas_workflow_diagram.pdf} +\end{minipage} +\end{figure} +\end{frame} +\note{ +\textbf{Flame with pandas workflow diagram (cont.)} + +\underline{Module Yellow:} Data transformation module + +g. Data transformations: simple arithmetic operations (+,-,*,/) on any available data + +h. Output a new data frame (in RAM) with for newly created variables (ratios, growth rates, sums, statistics, ...) + +i. Output a new HDF5 file to disk after data transformations are done + +\underline{Module Red:} Data preparation module + +j. Data preparation: depending on the type of analysis (agent-level, run-level, ensemble-of-runs, set-level, ensemble-of-sets) + +\underline{Module Green:} Plotting module + +k. Data visualization: produce plots depending on the type of visualization (views) and whether you want a single file (multiple lines per plot) or multiple files (single line per plot) +} + +%-------------------------------------------------------------------------- +\section{Data visualization} +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Time series plot: multiple agents, 1 variable, 1 set, 1 run\\(2D - time series of 20 agents)} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/plots/time_series/agent_level/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.45]{[Set2]_Bank_1_set_1_run_20_instance.png} +\end{minipage} +\end{figure} +\end{frame} +\note{ +This plot shows: + +- the data for this plot is 2D + +- the data is for 20 agents, 1 variable, 1 set, 1 run + +- we plot the time series for each agent individually. +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Time series plot: multiple agents, 1 variable, 1 set\\(3D - quantiles across agents + 20 runs)} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/plots/time_series/Bank_plots/set_2/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.45]{Bank_equity_1_set_20_run(set2_unstable_case).png} +\end{minipage} +\end{figure} +\end{frame} +\note{ +This plot shows: + +- the data for this plot is 3D + +- the data is for 20 agents, 1 variable, first gets collected in an ensemble distribution: all observations at one point in time are combined + +- this is done for 20 runs, so in principle the underlying data consists of 400 time series (20 runs, 20 agents). + +- and then we take the quantiles across these 400 data sets +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Time series plot: multiple agents, 1 variable, 2 sets\\(4D - quantiles across 20 runs)} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/plots/time_series/Bank_plots/set_1/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.45]{Bank_equity_2_set_20_runs_quantile.png} +\end{minipage} +\end{figure} +\end{frame} +\note{ +This plot shows: + +- the data for this plot is 4D + +- the same type of plot as in the previous slide, but now for 2 parameter sets. +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Time series plot: multiple types, multiple agents, 2 variables, multiple runs, 1 set (5D - time series of ensemble quantiles)} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/plots/time_series/Bank_Firm_joint_plots/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.45]{[set_2]_Bank_Firm_equity_1_set_20_runs_quantile.png} +\end{minipage} +\end{figure} +\end{frame} +\note{ +This plot shows ensemble quantiles over time: + +- the data for this plot is 5D + +- here we have multiple agent types: Bank and Firm + +- each type has multiple agents, so we collect their time series over 20 runs + +- then we consider the ensemble: 20 banks + 80 firms -> 2,000 data series in total + +- and plot the quantiles of Bank.equity (400 series) and Firm.equity (1600 series), at each point in time + +- the data for this plot is the same as for the boxplot, but we plot the entire time series evolution of the agent distribution of the variable, and we do this for two different agent types, and two different variables. +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Box plot: multiple agents, 1 variable, multiple runs, 1 set\\(3D - box plots of ensemble distribution across agents + 20 runs)} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/plots/time_series/Bank_plots/set_1/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.45]{Bank_equity_1_set_20_runs_boxplot.png} +\end{minipage} +\end{figure} +\end{frame} +\note{ +This plot shows box plots of an ensemble distribution of data: + +- the data for this plot is 3D + +- for multiple agents, we collect their data series across 20 runs + +- then we consider the ensemble: 400 data series in total + +- and we plot a box plot at selected points in time + +- each box plot shows the ensemble across all agents and across all 20 runs +} + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Some Performance Metrics (Memory consumption, 100 sets) } + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/Benchmarks/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.55]{Fig_100.png} +\end{minipage} +\end{figure} +\end{frame} + + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Some Performance Metrics (Memory consumption, 250 sets) } + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/Benchmarks/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.55]{Fig_250.png} +\end{minipage} +\end{figure} +\end{frame} + + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Some Performance Metrics (Memory consumption, 513 sets) } + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/Benchmarks/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.55]{Fig_513.png} +\end{minipage} +\end{figure} +\end{frame} + + +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Some Performance Metrics (Memory consumption, comparing three cases) } + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/Benchmarks/}} +% +\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.55]{Fig_all.png} +\end{minipage} +\end{figure} +\end{frame} + + +%-------------------------------------------------------------------------- + +\section{Summary} +%-------------------------------------------------------------------------- +\begin{frame}{}\small +\frametitle{Summary} + +\begin{enumerate}\itemsep2em + +\item Software library for ABM Data Analysis using Python pandas + +\item General-purpose tool for analysing ABM data + +\end{enumerate} +\vspace{0.7cm} +\textbf{Re-stating the important question: How to fit it all together (given the context of CONQUAIRE)!} +\newline + +\begin{enumerate}\itemsep2em + +\item How is it to be integrated in the Conquaire workflow; also what measures of Quality Control for our particular case? + +\item What are the requirements (expected of us) for the proposed Quality Control? + + +\end{enumerate} +\end{frame} +\note{no notes +} + +%-------------------------------------------------------------------------- + +\section{EndPage} +%-------------------------------------------------------------------------- +\begin{frame}{}\small +\vspace{1cm} +\begin{center} +\huge{Suggestions / Discussion / Questions ?} +\end{center} +\end{frame} +\note{no notes +} + + +%-------------------------------------------------------------------------- +% Appendix: Reserve slides +%-------------------------------------------------------------------------- + +\begin{frame}{}\small +\frametitle{Data Volume Table} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{./png/}} +% +%\hspace{-4cm} +\begin{minipage}{10cm} +\centering\leavevmode +\includegraphics[scale=0.4]{table_data_volumes.jpg} +\end{minipage} +\end{figure} +\end{frame} +\note{} + + +\end{document} + +%-------------------------------------------------------------------------- +% Appendix: Template slides +%-------------------------------------------------------------------------- +\begin{frame}{}\small +\frametitle{} + +\begin{enumerate}\itemsep2em + +\item + +\item + +\item + +\end{enumerate} +\end{frame} +\note{ +} + +%-------------------------------------------------------------------------- + +\begin{comment} +\begin{frame}{}\small +\vspace{-1cm} +\thispagestyle{empty} +\textbf{Title} + +\begin{figure}[hb!] +\centering\leavevmode +\graphicspath{{../png/}} +% +\begin{minipage}{16.5cm} +\hspace{-1cm} +% +\begin{minipage}{3cm} +\centering\leavevmode +%{\small $t=0$} +%\includegraphics[width=3cm]{cuttlefish_0000.pdf} +\end{minipage} +% +\end{minipage} +\caption{} +\end{figure} +\end{frame} +\end{comment} + + +\end{document} \ No newline at end of file diff --git a/docs/slides/Tasks.txt b/docs/slides/Tasks.txt new file mode 100644 index 0000000..9da13b2 --- /dev/null +++ b/docs/slides/Tasks.txt @@ -0,0 +1,14 @@ + +We started by separating the project into six tasks: + +Task 1: Reverse engineering SQL database files to XML files + +Task 2: Python pandas's Data Panel format and the HDF5 format + +Task 3: Descriptive statistics scripts + +Task 4: Data visualization scripts + +Task 5: Input Output Facilities + +Task 6: Data Transformations \ No newline at end of file diff --git a/docs/slides/logos/ETACE_new.png b/docs/slides/logos/ETACE_new.png new file mode 100644 index 0000000..1614991 Binary files /dev/null and b/docs/slides/logos/ETACE_new.png differ diff --git a/docs/slides/logos/UniBi_Logo_54.pdf b/docs/slides/logos/UniBi_Logo_54.pdf new file mode 100644 index 0000000..c664002 Binary files /dev/null and b/docs/slides/logos/UniBi_Logo_54.pdf differ diff --git a/docs/slides/logos/UniBi_Logo_54.png b/docs/slides/logos/UniBi_Logo_54.png new file mode 100644 index 0000000..8ea0766 Binary files /dev/null and b/docs/slides/logos/UniBi_Logo_54.png differ diff --git a/docs/slides/logos/dfg_logo_blau.jpg b/docs/slides/logos/dfg_logo_blau.jpg new file mode 100644 index 0000000..3302a90 Binary files /dev/null and b/docs/slides/logos/dfg_logo_blau.jpg differ diff --git a/docs/slides/logos/logo_h2020_big.jpg b/docs/slides/logos/logo_h2020_big.jpg new file mode 100644 index 0000000..f28c00e Binary files /dev/null and b/docs/slides/logos/logo_h2020_big.jpg differ diff --git a/docs/slides/logos/logo_h2020_big.pdf b/docs/slides/logos/logo_h2020_big.pdf new file mode 100644 index 0000000..5687722 Binary files /dev/null and b/docs/slides/logos/logo_h2020_big.pdf differ diff --git a/docs/slides/logos/logo_h2020_big.png b/docs/slides/logos/logo_h2020_big.png new file mode 100644 index 0000000..2322e14 Binary files /dev/null and b/docs/slides/logos/logo_h2020_big.png differ diff --git a/docs/slides/logos/unibi-logo-gruen.pdf b/docs/slides/logos/unibi-logo-gruen.pdf new file mode 100644 index 0000000..e6dd193 Binary files /dev/null and b/docs/slides/logos/unibi-logo-gruen.pdf differ diff --git a/docs/slides/logos/unibi-logo-gruen.png b/docs/slides/logos/unibi-logo-gruen.png new file mode 100644 index 0000000..651f46e Binary files /dev/null and b/docs/slides/logos/unibi-logo-gruen.png differ diff --git a/docs/slides/png/Benchmarks/Fig_100.png b/docs/slides/png/Benchmarks/Fig_100.png new file mode 100644 index 0000000..78092ec Binary files /dev/null and b/docs/slides/png/Benchmarks/Fig_100.png differ diff --git a/docs/slides/png/Benchmarks/Fig_250.png b/docs/slides/png/Benchmarks/Fig_250.png new file mode 100644 index 0000000..05135eb Binary files /dev/null and b/docs/slides/png/Benchmarks/Fig_250.png differ diff --git a/docs/slides/png/Benchmarks/Fig_513.png b/docs/slides/png/Benchmarks/Fig_513.png new file mode 100644 index 0000000..f63dc4b Binary files /dev/null and b/docs/slides/png/Benchmarks/Fig_513.png differ diff --git a/docs/slides/png/Benchmarks/Fig_all.png b/docs/slides/png/Benchmarks/Fig_all.png new file mode 100644 index 0000000..3761f6d Binary files /dev/null and b/docs/slides/png/Benchmarks/Fig_all.png differ diff --git a/docs/slides/png/conquaire_kick-off-meeting-slides.pdf b/docs/slides/png/conquaire_kick-off-meeting-slides.pdf new file mode 100644 index 0000000..964948d Binary files /dev/null and b/docs/slides/png/conquaire_kick-off-meeting-slides.pdf differ diff --git a/docs/slides/png/data_processing.jpg b/docs/slides/png/data_processing.jpg new file mode 100644 index 0000000..f7d4e9a Binary files /dev/null and b/docs/slides/png/data_processing.jpg differ diff --git a/docs/slides/png/data_visualization.jpg b/docs/slides/png/data_visualization.jpg new file mode 100644 index 0000000..8bc66b0 Binary files /dev/null and b/docs/slides/png/data_visualization.jpg differ diff --git a/docs/slides/png/flame.png b/docs/slides/png/flame.png new file mode 100644 index 0000000..d5480ea Binary files /dev/null and b/docs/slides/png/flame.png differ diff --git a/docs/slides/png/flame_pandas_workflow_diagram.pdf b/docs/slides/png/flame_pandas_workflow_diagram.pdf new file mode 100644 index 0000000..5bb77bb Binary files /dev/null and b/docs/slides/png/flame_pandas_workflow_diagram.pdf differ diff --git a/docs/slides/png/flame_pandas_workflow_diagram.xml b/docs/slides/png/flame_pandas_workflow_diagram.xml new file mode 100644 index 0000000..70f821e --- /dev/null +++ b/docs/slides/png/flame_pandas_workflow_diagram.xml @@ -0,0 +1 @@ +7V1tj9u4Ef41C1w/xBBFipQ+5qXpFei1AXJAr58C2ZJ3hfNarqxNNv31lSxRFocjW5aHtrPZBe6ylrV6efhw3jgzvOPvH5//VsSbh9/yJF3d+V7yfMc/3Pk+4yyq/qmPfG+OhJ7fHLgvsqQ9aX/gc/a/tD3otUefsiTdGieWeb4qs415cJGv1+miNI7FRZF/M09b5ivzrpv4PrUOfF7EK/vov7OkfGiPMhntv/g1ze4f9K1l0L7gPF78eV/kT+v2hnc+X+5+mq8fY32x9k23D3GSf+sd4n+94++LPC+b3x6f36erGlyNW/N3Hwe+7R68SNflmD9oH/trvHpq3/23OFvf+bJ68Go0+LvHPHmqYGketvyuEaqee1P/usgfN/l6d7N38Sq7X1fHVumy/rjdxItsff+P3acPXFaHltlq9T5f5cXuIjzd/dSnlkX+Z9r7hkse8aT+i3xdfm5vy9rPvfO86uf9++q4/d4tFF/Tokyfe4daHP6W5o9pWXyvTtHfinZMWtJqEn/bMyCI2lMe+oPve5p6Le3uu2vvka9+acEfGDnFraGwUP/2kJXp5wrX+vO3au5Vb/5QPq5acJJ4+5AmGikT6yRIw0RgWIf+nMt6dO6LOMlSA99IJZ5SHb4WmAjkg/j6vjDwldKzABYowMojwFdf2RG+y3CRLhYYvvMwEIFHhKE0MVQisDEUvo1hNYhnQ+hjDO0EBf7rp1Ulyd77d2+9q8uUTlZcUKYoIFL8kFvjFXrSHi8/Cgg4L4RbzkdLVX93QH4TYOhLCGIUWiBKDxEcfkAgOLTOd4ThIknn4dw5hpCInElbu4UYhswnIGJo2xmUIKasUm8KAzGSiseSRvgqLgCIvg0iw2Yzi0hAZC8ARMkCCKKyQWQMA5HCzArdmgGXAVFEagQTcRApmKjcysTLgMjDMSB63BGI6kZE4pl6JRqDoSsiyhux6c/DUD+0xtDTBvxRx0hSOEZKOgVxhONJMJmDQAIQpQ0i46j7HlGAGLwAEIWeqAdB1HgZIHISJrp1Vy4Dog/DSDiIAgORwsAR51uJh8NxmKt80MNZpMFySSMrIw+AK6TtUPvYNJcEARDGzrd7oIpZLn1cxSRyLoOBsNxymUSSSIUrBQxyTzEbUoVJzohAcgZuVXgSp+ESxVcuwnRORMvqNQwM+cjIpvQJxGbgNnB8GSOoiz10wXcbQlR9C4oYT+DWs74MhHwMhNIZhG7dmQuZ4vyaLGTs6ktAFBhay2jIEgVGw5ACQu8FaugImOVCz7W+ghaYgg4oDEovfHmQhiw0IO3W43uQcu7bkHYpCudBenVv8fha73nqHAiBAIE3wBjLxPnwan/LlSo6zl4KMapXW3UKjV586K8copM+kgTa/OoxIccEVSBwKRF4fb3gYMbcCOBFF9JX9QJ3kn2tfr0vd28p48ca1PV8u9l99uxD1udV86cfzQtNurZ16O/Vdf+1Q76CiPBW8X09kL5Xft+k26ELA/bpZIF1Xlb/vDsW0WhSzFjNuZok2SJevW1TC8p89wxoosHv9Xcf3iB5BsvlYlH5v47XbZnnm3HNTs8b8XXUtTyfplg4rhmcOg5kjIf871Ouv3jToP22OoGJzfP+Sz2Yv374WClAr0jj5M0u2UwPeHPZgRHfZdV1EuXIgFNIYG6uD3E7H6wzGwzkKbwpC/lNkS/SbTU76qTHZXb/VMRlllfgeRUxU2zeBfVU01M2Sbdltm7/5H39dOac8+6CDyNn3qXHIQD5HwqxJbBxoMgaw9QgJuF2h+YFPOJIDusLVUJqDeZUnwzV3ftn/DCC9drOgTDdLWTeoyuaFCEA5Ujivq1n+/ZWJa3wwOIds40xhk1xCsht7/ZDXMbLIn5Mv7xFpuwv9d/O0ln1z7t4/Wf1z19uVHD6AtgOHmY7IAtRFJLTjsPsYcUkYQ/Wvz4V+baMyxuGVrDoKLQ6S5EaWR2BQKH95wC0O3X/pVb2X55vFdQg4kdBRW1dElSxJYBBVQ/0qqGdv8ZFFs9XO6uqzI8o4wvaDNihTZ2eXX/zkdKVbDAcYbRsV1ltDPzoBsslPMEQLDH6wtaR2BJjQBBQQ9cliM3go9PqgB08wJv+sXgdr75vs23r8ZzIuLEPPPz+r7wejHCANTeOlAXpTE1q24/ZfnZSqdI3O126O17cddVdVzU3gCWHrUsyxJIjAcl2gvsgHZfyDYr+4Gy4MXB5YC9YouAKihId2+ObAu4ftwuuGUQTSN1Jt+5tgEuht2zfToe99vrg2k6aXszt0q+QEjGdaEUdZkTy2R6f6jLrOojkFU/rW0IK+lwVUogJhFTfUCDl2+5BD6ltelOcCpSASCGcQiYdCVK+hdSlDCg6u9L71DhGI2xFRx7cLnpfZo/pm21aZMPLYkTu4Tx/ftM4g/XiQFouZgcWBM4w/X9KGzfyzWwT1hUJGpENxMiliGwcWMZz6r1dZy7/66ncPNVTt276kRbnBEemR1XM9fFpUuOnnCkhrEDpyhmOxgA5wUyx3cHf+xL42qZiaMZHGUeSVrCcFRK1bnuBv2bbMr+vHJWrIxN4sDJb5+4dMw0pWGO7cO/y50aZXhkXHo1hjKMcBmxlLV3F9ePdBDgKrPNedDrxSyllN+Zpax5vinxTa5fXrK0j9h4Wr/MQp57C3uMjkosv1b6LAEwB0n9CJAMO7VzCVEgApq324PqeVxbxervMixvQg0Deh7rvihFtCxCRRlDJx4eTVy4ml7qhaDPdnPrMtew7eJ+fUvgxj5t6FSvRdZZApYVDj4Tr9Ft1INEpElefpLCNEDpJfcxYJZikaHX4iPlAlxzQTdEK8jMu0xvPLsmVzIfP1kdlyOvc3k1Z327Fhs5tiqoJ3Ral74L2s169atAgwbyetr7yvA9Bv6EwREIbWEUPhVEo7BSOz0+Pj3F9lX3fxl2jxhvo+Hq8ayOBGA5BBVBgU9nH6lQrP5FgPC7g/9lpWcBwJYx5Hl+o7lnJNIHg1xDqgPNp+ksitHmte28Z1pdPwGrM9bywC7DVYu0Kgf9ph8Ai4yuFYa4jmhKGeLEkfSyGBfMcEnN0TUZg1mR4iw63/cGdplss7qzajVmPG2bZRnfYejJstg09ve6+r+8rLvuo4+X96wu8vsCZL3CqyXCSj2BUvBOIQdhc08c6qaBtPwgSE4NhVX4ZsWeRbnaEAhhbEA444CzJo44m8esLvL7AmS/wY8lBCTYOwfr/43KQwKUJ7FXsup7u7x/vdtWJ3sen3el1Bd7VI0y+NNcdJRJZ9vVWTEayNUWpvB1hytfpm7zNxzoHGsQJwSvESbdMgLkTgtleSMCRpD1BsO4obTfkMV5//5HhBCk6QtrWTKCXWw04CTq3jmnPnK6Tt/XeYdWnxSrebrOFCVsDj94PzD8ES5oY+4sNr20HIAZpb4WAdck65Oe29/qUZzsdpROBNLDtjWDn1m3+VCzS9o/2sFrXsSr1ObhQGRf3aWldqAJ2l/ijT9vUJ2zHP6++z364myvuB78DdxwfRvS3OsaHatCL739UH7xZoD/+p/7oiCvShEQh04eEKwJoESaBOzGaLPBCcE+jk8liTPcTRxzLTrjxEVfmYpuy19EdjbieHqeOuIQpdbAXHZF4sO7TPjCZeBjR8vIa6iIEhLA70xARAm4XNZUQlhGlrQBiQnQrGo4IMWbXoAtJjwaxdmTFqToELEEjFi0JgToDQ7tscA17tA6JGPrER/gzZYhH9AA/OsTPWdmMsOdF7efdEM+E57efP6VFVj1WWgwO/Z4oBk12rBmkRQNoa93ZTNFJf+OZYqqE0JmogaXtPvCeRjMFlnHDLUmoTFPASP3AQ88Fz+dttTOZaPIpRZPNOCeGjUkuvRUWvdsDsdfBj9PJpWZ1g/0oaP9vXlZFs953XLlRctbbtMKQjEkj2s+fbvVMkmaG6FInsgsk9iBBMEfsUpPZZYouEbgxm+EDC05sJY3Yv+kcK2mvXoOecj3R2O4rSh3RM2yqUzWlMm0qpNSDiG4esKmgLTSWbtwLZ7L3A7oFcC+Y8f4PiL/RmVxdIec5hBm94Y+3+0HJgjQUVWksU+8sbTeSAUW6isvsa2pc/Azn3Z+s5Kwrhdzd0BPICtdD38kbw5Q/qLgAKUxx06wlDUqSNzycMR4FXIRhpUvAdK8mSzDzhAwYk5Xn77kKFfsg9Krg4tBo39/aCE+Mo9PJzj+4kWonAJVa67o1kOg1ZppB/ok2dl9Vner9Bwrg5IhCEuyuooBQGy2QwAPDAhiqaCJ43DaqQkYfpGHrdPrUoQVuUkh0hhKMLdgrfUnlrWKttVU0Hy8gl8s48A7qxjNYam3LjC27ktCU6YISTa+JQU4B3EEJG8MRqk20Te15anMEKw5xwEV8gAPVh+zOREMBuLdAV3d3KgcCsE+HdSFKDvi3z4EhdXeK8TTMDwH4wV2tjO07p3VtdydaQ2HkGV4V2HQVbitGpNqs59c3olNu9Hb8xcjo+4Oa83yOQsmC9Gum4WgIVgWt1pljKapA86Uu8kZMSuuB2/vQcZIg28c5JwdjWXu2VqaeMBm7N/1GE9aId7XCwAyv8hOZDdISmJbGDqSvh0e8T6d2AFxE+DBk8hYE04TwTnqwU8+XktpZIfB1LybPvZnygUxvF17dyHSYFoFsa0kk0yVsmzzV7JBq5oeBqnyd5v8me7iY9b90419bLxPSJvMxpM/zzXH2bPLJyCQf14tELoxeEHfxJloUzAOhuTAEV6Kzc2FzBWK5qF1QUo6lcmCns+FwTE/2CeED2cecib4AbEcbOQseKgVco67vx8nsi/hMMKZkJbeDMOx6YHReI4tm+2853CybipowBU6/Dx017ZjQvgi+3kgr25bZ4gYaoIDlZxkiXZBdNUBhSHNXq+jq96H2YVYF1pHqWKO+XzdUGV/ACgaqwrfE0iy0xFjn6xSIl/aQru9fVKNWCwOrDcBjliT1bdDhH2rFQsAFaEqFke1EdMTucwFmdU3jgm3kNruqzr7HFcY/Af6wS0j1g+YEHCmx80lGw07N75rF/DwDArZBEtLuDqWQEWARxQjYBnTdovfLvr/rTzMOIRgHdO8MZwPBMSuz7aX0tBmqQz61EH+45Nhr/5tSemw84MsnSsChBBVXk6C62/mLdoAFNKSRfbrIHGBzNS9kwDcYH1I/cqFLlsB1y2S3TBMdF9flFPtlw0OZoWPJZYbDm0W4QcIpz8wL6crCHBAuULPaBfWjxmE11zcjP5w1Lmrz/cQKiTpIMuvdRIH5JDxl+stnUnVkqCbyicOBaAP0C3ehq82n2SZZjvTwrt/XbekPhKLkXAZyaBonkZRE9i/zLDbadpejrtEMafPuljI1P5hzgsgXTRCJiGNHXQG7PgcXJciX51cRclJ36gAkYXVL1T2K6M4qRosRCt8NaT3vniL+K0NOkSGgox4qQ7CezyQyBOlffnPm91hDejh+wszlG+Gs/C4EvehZ5yCeahhHcNc560qE2ZtIJ/YXyAIgiYVvC2IqFkCzQAdDTmYBgyyAV7qos36gl3Yb9xtsMFiX1cFon9X8fYPucDuyiWbTuB2JPLbPVEcdi/v5L97ONaq3lO399hf76R7jzaberPOkzvBg0py0nImWiHX7DDgKGTLQ7otxhvg4aJAQ+OLT9A9BCyTaFIY7B7nhIZAHztJkYJfyyfUjLAhn+yyErstvJ4ZYcIE0BQUWZPXbDEY2wfmyLX4ffEl23vk1DIDtZ4aJxCWzxroK1PGK+FBjDnF+liPIAm6SxgenVbetu84+Q1ryU9WfWoGgqXXuLJwFQbgvqQXCV3kzFsp9Se2oeTWFaXrKntNLW2BLeB+zVbnb47ppP+vpDbHm9chllTZ8HlTtgOintPSFZddBGiYCI33ozzmV6waLfAUSjBeY60axWytDd4U4UVY46QlpZps76z3RtVzW6MNLjG8IBy7krqEXQ1vY38CgwfoJz9WoSZDxOHnUrAu5FJbDoXiHwnKRr5MMJP29LIEZIKEuhwLTcbtVklZAU8Usc7YQDNMiAz61Gx68kAQXopyxY9dGpm4jd3JkfKmnejetXzcoP5T92bWWMrbxdBUH1wLmliOgPdkThaDfhqdFTueY2Xkq08oz9UwyO3eeGwFx5qqFoPWZgMlso6vU1MEqNXGJKjUJWghzTlxBJDEv8MZof76qBI1/GbbLA41xC8ZLTE2LghcKYLnJRcP+0h+pTK0NVrfx13SnVOutaZJsu4iL5EhEfdoitHVnqH6nbvX6qotr+iFFxbpwinrPHoZs2uM8Va63P/fsIfhhuHDt3AXm+UC0Yhly6ObrJFabstWXsREWrJ0DAzih2NDCBkFw2P0F+w0pRBFxbK0Ndk+ZBpa9tvB5EZeVPdgsu14ZHOnDUlhuVytESC4dSWyAok2w89afAwsvvBsAaznlSBu8I8nmoZr11yJgiJBdpPengOX6U9tO8Mpkj3o/oLGsB3fmI7LhreePaHvts5C+s9CFutaqO9A7QN2BvinisO86hu92cYU5Bxrxcdtz4DB1FWyJfPMzwkQxksSr6CEWrj9xRgz5occpd2CRBYRsw0uFbCWsUJ8aspVQ2RKGbEOC8Pz4xJ7pSrNZdzoQ6gKjjOwMS9SVxAODE02MNqgwOlCDJUPaEqyzOOJwE7RzZjYw7H3mKsFLgY1EFJTRoxe9YaWdB1vaUHVCBzfST0wn7e2lHsMJ/Kz7zlzZwRGRmX980Z2QWUQQ6T24JDFQ/3rOrIJ9bp0lJVibP02dVUKpA2mTSqqZ1y+aJdKl1ccir/MK9qdXBH74LU/S+oz/Aw== \ No newline at end of file diff --git a/docs/slides/png/hdf5_data_groups.jpg b/docs/slides/png/hdf5_data_groups.jpg new file mode 100644 index 0000000..b8a10e3 Binary files /dev/null and b/docs/slides/png/hdf5_data_groups.jpg differ diff --git a/docs/slides/png/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files.png b/docs/slides/png/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files.png new file mode 100644 index 0000000..02d836e Binary files /dev/null and b/docs/slides/png/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files.png differ diff --git a/docs/slides/png/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files_with_file_size.png b/docs/slides/png/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files_with_file_size.png new file mode 100644 index 0000000..c9367ef Binary files /dev/null and b/docs/slides/png/hdf5_snapshots/Fig_0_list_of_separate_hdf5_files_with_file_size.png differ diff --git a/docs/slides/png/hdf5_snapshots/Fig_1_individual_hdf5_corresponding_to_db_agent_1.png b/docs/slides/png/hdf5_snapshots/Fig_1_individual_hdf5_corresponding_to_db_agent_1.png new file mode 100644 index 0000000..4e8fa45 Binary files /dev/null and b/docs/slides/png/hdf5_snapshots/Fig_1_individual_hdf5_corresponding_to_db_agent_1.png differ diff --git a/docs/slides/png/hdf5_snapshots/Fig_2_individual_hdf5_corresponding_to_db_agent_2.png b/docs/slides/png/hdf5_snapshots/Fig_2_individual_hdf5_corresponding_to_db_agent_2.png new file mode 100644 index 0000000..f0f669f Binary files /dev/null and b/docs/slides/png/hdf5_snapshots/Fig_2_individual_hdf5_corresponding_to_db_agent_2.png differ diff --git a/docs/slides/png/hdf5_snapshots/Fig_3_hdf5_file_per_agent_showing_values.png b/docs/slides/png/hdf5_snapshots/Fig_3_hdf5_file_per_agent_showing_values.png new file mode 100644 index 0000000..d028d6c Binary files /dev/null and b/docs/slides/png/hdf5_snapshots/Fig_3_hdf5_file_per_agent_showing_values.png differ diff --git a/docs/slides/png/hdf5_snapshots/fig_4_list_of_hdf5_files_separated-per-agent-type.png b/docs/slides/png/hdf5_snapshots/fig_4_list_of_hdf5_files_separated-per-agent-type.png new file mode 100644 index 0000000..dcaf613 Binary files /dev/null and b/docs/slides/png/hdf5_snapshots/fig_4_list_of_hdf5_files_separated-per-agent-type.png differ diff --git a/docs/slides/png/panda-eat.jpg b/docs/slides/png/panda-eat.jpg new file mode 100644 index 0000000..cba363b Binary files /dev/null and b/docs/slides/png/panda-eat.jpg differ diff --git a/docs/slides/png/plots/time_series/Bank_Firm_joint_plots/[set_2]_Bank_Firm_equity_1_set_20_runs_quantile.png b/docs/slides/png/plots/time_series/Bank_Firm_joint_plots/[set_2]_Bank_Firm_equity_1_set_20_runs_quantile.png new file mode 100644 index 0000000..2018e64 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_Firm_joint_plots/[set_2]_Bank_Firm_equity_1_set_20_runs_quantile.png differ diff --git a/docs/slides/png/plots/time_series/Bank_Firm_joint_plots/set_3_error_screenshot.png b/docs/slides/png/plots/time_series/Bank_Firm_joint_plots/set_3_error_screenshot.png new file mode 100644 index 0000000..f8de410 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_Firm_joint_plots/set_3_error_screenshot.png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_1_run_quantile_(agent).png b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_1_run_quantile_(agent).png new file mode 100644 index 0000000..52cab53 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_1_run_quantile_(agent).png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_20_runs_boxplot.png b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_20_runs_boxplot.png new file mode 100644 index 0000000..b32e992 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_20_runs_boxplot.png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_20_runs_quantile.png b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_20_runs_quantile.png new file mode 100644 index 0000000..65e4b83 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_1_set_20_runs_quantile.png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_2_set_20_runs_quantile.png b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_2_set_20_runs_quantile.png new file mode 100644 index 0000000..b0fad9c Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_1/Bank_equity_2_set_20_runs_quantile.png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run(set2_unstable_case).png b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run(set2_unstable_case).png new file mode 100644 index 0000000..dfce874 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run(set2_unstable_case).png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run_(set2)_20_agents_dotted.png b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run_(set2)_20_agents_dotted.png new file mode 100644 index 0000000..8e7945e Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run_(set2)_20_agents_dotted.png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run_(set2)_20_agents_solid.png b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run_(set2)_20_agents_solid.png new file mode 100644 index 0000000..4a40d12 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_1_run_(set2)_20_agents_solid.png differ diff --git a/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_20_run(set2_unstable_case).png b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_20_run(set2_unstable_case).png new file mode 100644 index 0000000..c6869d2 Binary files /dev/null and b/docs/slides/png/plots/time_series/Bank_plots/set_2/Bank_equity_1_set_20_run(set2_unstable_case).png differ diff --git a/docs/slides/png/plots/time_series/agent_level/[Set2]_Bank_1_set_1_run_20_instance.png b/docs/slides/png/plots/time_series/agent_level/[Set2]_Bank_1_set_1_run_20_instance.png new file mode 100644 index 0000000..b9b2a86 Binary files /dev/null and b/docs/slides/png/plots/time_series/agent_level/[Set2]_Bank_1_set_1_run_20_instance.png differ diff --git a/docs/slides/png/plots/time_series/agent_level/screenshot_one_line.png b/docs/slides/png/plots/time_series/agent_level/screenshot_one_line.png new file mode 100644 index 0000000..9a168e1 Binary files /dev/null and b/docs/slides/png/plots/time_series/agent_level/screenshot_one_line.png differ diff --git a/docs/slides/png/plots/time_series/agent_level/zoomed_screenshot_one_line.png b/docs/slides/png/plots/time_series/agent_level/zoomed_screenshot_one_line.png new file mode 100644 index 0000000..3a50b61 Binary files /dev/null and b/docs/slides/png/plots/time_series/agent_level/zoomed_screenshot_one_line.png differ diff --git a/docs/slides/png/python-pandas-screenshot.png b/docs/slides/png/python-pandas-screenshot.png new file mode 100644 index 0000000..f05ca72 Binary files /dev/null and b/docs/slides/png/python-pandas-screenshot.png differ diff --git a/docs/slides/png/rubiks-mirror-cube-mixed-up.jpg b/docs/slides/png/rubiks-mirror-cube-mixed-up.jpg new file mode 100644 index 0000000..2a6fe2c Binary files /dev/null and b/docs/slides/png/rubiks-mirror-cube-mixed-up.jpg differ diff --git a/docs/slides/png/rubiks-mirror-cube-regular.jpg b/docs/slides/png/rubiks-mirror-cube-regular.jpg new file mode 100644 index 0000000..895c71b Binary files /dev/null and b/docs/slides/png/rubiks-mirror-cube-regular.jpg differ diff --git a/docs/slides/png/set_run_hierarchy.png b/docs/slides/png/set_run_hierarchy.png new file mode 100644 index 0000000..713263d Binary files /dev/null and b/docs/slides/png/set_run_hierarchy.png differ diff --git a/docs/slides/png/set_run_hierarchy_2.png b/docs/slides/png/set_run_hierarchy_2.png new file mode 100644 index 0000000..ec2550f Binary files /dev/null and b/docs/slides/png/set_run_hierarchy_2.png differ diff --git a/docs/slides/png/sql_table_snapshot.jpg b/docs/slides/png/sql_table_snapshot.jpg new file mode 100644 index 0000000..d914066 Binary files /dev/null and b/docs/slides/png/sql_table_snapshot.jpg differ diff --git a/docs/slides/png/table_data_volumes.jpg b/docs/slides/png/table_data_volumes.jpg new file mode 100644 index 0000000..b6e7515 Binary files /dev/null and b/docs/slides/png/table_data_volumes.jpg differ diff --git a/docs/slides/png/workflow_cheops.jpg b/docs/slides/png/workflow_cheops.jpg new file mode 100644 index 0000000..f2fddf7 Binary files /dev/null and b/docs/slides/png/workflow_cheops.jpg differ diff --git a/docs/slides/png/workflow_data_conversion/workflow_data_conversion.dia b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.dia new file mode 100644 index 0000000..bd3c33f Binary files /dev/null and b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.dia differ diff --git a/docs/slides/png/workflow_data_conversion/workflow_data_conversion.eps b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.eps new file mode 100644 index 0000000..2db0ff9 --- /dev/null +++ b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.eps @@ -0,0 +1,6133 @@ +%!PS-Adobe-2.0 EPSF-2.0 +%%Title: /media/sander/DataStorage1/GIT/GitLab/ETACE-Conquaire/Slides/Slides-ETACE-Retreat-2017/src/png/workflow_data_conversion/workflow_data_conversion.dia +%%Creator: Dia v0.97.2 +%%CreationDate: Tue Mar 21 16:35:34 2017 +%%For: sander +%%Orientation: Portrait +%%Magnification: 1.0000 +%%BoundingBox: 0 0 1374 340 +%%BeginSetup +%%EndSetup +%%EndComments +%%BeginProlog +[ /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef +/.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef +/.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef +/.notdef /.notdef /space /exclam /quotedbl /numbersign /dollar /percent /ampersand /quoteright +/parenleft /parenright /asterisk /plus /comma /hyphen /period /slash /zero /one +/two /three /four /five /six /seven /eight /nine /colon /semicolon +/less /equal /greater /question /at /A /B /C /D /E +/F /G /H /I /J /K /L /M /N /O +/P /Q /R /S /T /U /V /W /X /Y +/Z /bracketleft /backslash /bracketright /asciicircum /underscore /quoteleft /a /b /c +/d /e /f /g /h /i /j /k /l /m +/n /o /p /q /r /s /t /u /v /w +/x /y /z /braceleft /bar /braceright /asciitilde /.notdef /.notdef /.notdef +/.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef +/.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef +/.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef /.notdef +/space /exclamdown /cent /sterling /currency /yen /brokenbar /section /dieresis /copyright +/ordfeminine /guillemotleft /logicalnot /hyphen /registered /macron /degree /plusminus /twosuperior /threesuperior +/acute /mu /paragraph /periodcentered /cedilla /onesuperior /ordmasculine /guillemotright /onequarter /onehalf +/threequarters /questiondown /Agrave /Aacute /Acircumflex /Atilde /Adieresis /Aring /AE /Ccedilla +/Egrave /Eacute /Ecircumflex /Edieresis /Igrave /Iacute /Icircumflex /Idieresis /Eth /Ntilde +/Ograve /Oacute /Ocircumflex /Otilde /Odieresis /multiply /Oslash /Ugrave /Uacute /Ucircumflex +/Udieresis /Yacute /Thorn /germandbls /agrave /aacute /acircumflex /atilde /adieresis /aring +/ae /ccedilla /egrave /eacute /ecircumflex /edieresis /igrave /iacute /icircumflex /idieresis +/eth /ntilde /ograve /oacute /ocircumflex /otilde /odieresis /divide /oslash /ugrave +/uacute /ucircumflex /udieresis /yacute /thorn /ydieresis] /isolatin1encoding exch def +/cp {closepath} bind def +/c {curveto} bind def +/f {fill} bind def +/a {arc} bind def +/ef {eofill} bind def +/ex {exch} bind def +/gr {grestore} bind def +/gs {gsave} bind def +/sa {save} bind def +/rs {restore} bind def +/l {lineto} bind def +/m {moveto} bind def +/rm {rmoveto} bind def +/n {newpath} bind def +/s {stroke} bind def +/sh {show} bind def +/slc {setlinecap} bind def +/slj {setlinejoin} bind def +/slw {setlinewidth} bind def +/srgb {setrgbcolor} bind def +/rot {rotate} bind def +/sc {scale} bind def +/sd {setdash} bind def +/ff {findfont} bind def +/sf {setfont} bind def +/scf {scalefont} bind def +/sw {stringwidth pop} bind def +/tr {translate} bind def + +/ellipsedict 8 dict def +ellipsedict /mtrx matrix put +/ellipse +{ ellipsedict begin + /endangle exch def + /startangle exch def + /yrad exch def + /xrad exch def + /y exch def + /x exch def /savematrix mtrx currentmatrix def + x y tr xrad yrad sc + 0 0 1 startangle endangle arc + savematrix setmatrix + end +} def + +/mergeprocs { +dup length +3 -1 roll +dup +length +dup +5 1 roll +3 -1 roll +add +array cvx +dup +3 -1 roll +0 exch +putinterval +dup +4 2 roll +putinterval +} bind def +/dpi_x 300 def +/dpi_y 300 def +/conicto { + /to_y exch def + /to_x exch def + /conic_cntrl_y exch def + /conic_cntrl_x exch def + currentpoint + /p0_y exch def + /p0_x exch def + /p1_x p0_x conic_cntrl_x p0_x sub 2 3 div mul add def + /p1_y p0_y conic_cntrl_y p0_y sub 2 3 div mul add def + /p2_x p1_x to_x p0_x sub 1 3 div mul add def + /p2_y p1_y to_y p0_y sub 1 3 div mul add def + p1_x p1_y p2_x p2_y to_x to_y curveto +} bind def +/start_ol { gsave 1.1 dpi_x div dup scale} bind def +/end_ol { closepath fill grestore } bind def +28.346000 -28.346000 scale +-27.200000 -38.479875 translate +%%EndProlog + + +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 27.250000 26.850000 m 27.250000 30.950000 l 33.200000 30.950000 l 33.200000 26.850000 l f +0.000000 0.000000 0.000000 srgb +n 27.250000 26.850000 m 27.250000 30.950000 l 33.200000 30.950000 l 33.200000 26.850000 l cp s +0 slc +0 slj +[] 0 sd +n 27.845000 26.850000 m 27.845000 30.950000 l s +0 slc +0 slj +[] 0 sd +n 27.250000 27.260000 m 33.200000 27.260000 l s +gsave 28.565000 28.905000 translate 0.035278 -0.035278 scale +start_ol +2624 3392 moveto +2624 2944 lineto +2352 3073 2111 3136 conicto +1870 3200 1645 3200 conicto +1255 3200 1043 3052 conicto +832 2904 832 2631 conicto +832 2402 969 2285 conicto +1107 2169 1491 2097 conicto +1773 2039 lineto +2308 1937 2562 1678 conicto +2816 1420 2816 986 conicto +2816 469 2468 202 conicto +2121 -64 1450 -64 conicto +1197 -64 911 0 conicto +626 65 320 192 conicto +320 704 lineto +613 513 893 416 conicto +1174 320 1445 320 conicto +1857 320 2080 485 conicto +2304 650 2304 955 conicto +2304 1221 2148 1371 conicto +1992 1522 1636 1597 conicto +1352 1652 lineto +807 1756 563 1978 conicto +320 2201 320 2597 conicto +320 3056 654 3320 conicto +989 3584 1576 3584 conicto +1828 3584 2089 3536 conicto +2351 3488 2624 3392 conicto +end_ol grestore +gsave 28.972119 28.905000 translate 0.035278 -0.035278 scale +start_ol +1890 3200 moveto +1375 3200 1071 2813 conicto +768 2426 768 1759 conicto +768 1094 1071 707 conicto +1375 320 1890 320 conicto +2406 320 2707 707 conicto +3008 1094 3008 1759 conicto +3008 2426 2707 2813 conicto +2406 3200 1890 3200 conicto +2553 67 moveto +3181 -640 lineto +2607 -640 lineto +2087 -55 lineto +2009 -59 1968 -61 conicto +1927 -64 1889 -64 conicto +1146 -64 701 431 conicto +256 927 256 1759 conicto +256 2593 701 3088 conicto +1146 3584 1890 3584 conicto +2632 3584 3076 3088 conicto +3520 2592 3520 1758 conicto +3520 1146 3271 710 conicto +3023 274 2553 67 conicto +end_ol grestore +gsave 29.476647 28.905000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 384 lineto +2688 384 lineto +2688 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 29.833809 28.905000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 30.036116 28.905000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.528150 28.905000 translate 0.035278 -0.035278 scale +start_ol +960 1728 moveto +960 384 lineto +1696 384 lineto +2071 384 2251 550 conicto +2432 716 2432 1057 conicto +2432 1401 2251 1564 conicto +2071 1728 1696 1728 conicto +960 1728 lineto +960 3136 moveto +960 2112 lineto +1639 2112 lineto +1975 2112 2139 2238 conicto +2304 2365 2304 2624 conicto +2304 2881 2139 3008 conicto +1975 3136 1639 3136 conicto +960 3136 lineto +448 3520 moveto +1673 3520 lineto +2222 3520 2519 3300 conicto +2816 3080 2816 2674 conicto +2816 2360 2658 2174 conicto +2500 1989 2193 1943 conicto +2549 1866 2746 1621 conicto +2944 1376 2944 1009 conicto +2944 526 2625 263 conicto +2306 0 1718 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.967739 28.905000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 31.170046 28.905000 translate 0.035278 -0.035278 scale +start_ol +2624 2688 moveto +2624 0 lineto +2176 0 lineto +2176 2368 lineto +960 2368 lineto +960 0 lineto +512 0 lineto +512 2368 lineto +64 2368 lineto +64 2688 lineto +512 2688 lineto +512 2871 lineto +512 3301 718 3506 conicto +924 3712 1351 3712 conicto +1792 3712 lineto +1792 3328 lineto +1376 3328 lineto +1142 3328 1051 3237 conicto +960 3147 960 2912 conicto +960 2688 lineto +2624 2688 lineto +2176 3712 moveto +2624 3712 lineto +2624 3136 lineto +2176 3136 lineto +2176 3712 lineto +end_ol grestore +gsave 31.572163 28.905000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 31.749492 28.905000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 32.144118 28.905000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 28.837500 29.705000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 29.087267 29.705000 translate 0.035278 -0.035278 scale +start_ol +1388 3712 moveto +2624 3712 lineto +2624 0 lineto +2176 0 lineto +2176 3328 lineto +1384 3328 lineto +1145 3328 1052 3237 conicto +960 3147 960 2912 conicto +960 2688 lineto +1664 2688 lineto +1664 2368 lineto +960 2368 lineto +960 0 lineto +512 0 lineto +512 2368 lineto +64 2368 lineto +64 2688 lineto +512 2688 lineto +512 2871 lineto +512 3310 722 3511 conicto +932 3712 1388 3712 conicto +end_ol grestore +gsave 29.489384 29.705000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 29.881513 29.705000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 30.131279 29.705000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 30.333586 29.705000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 30.583353 29.705000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 30.975482 29.705000 translate 0.035278 -0.035278 scale +start_ol +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +896 2304 moveto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +end_ol grestore +gsave 31.382601 29.705000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 31.559930 29.705000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 31.954556 29.705000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 69.650000 26.540000 m 69.650000 30.640000 l 75.600000 30.640000 l 75.600000 26.540000 l f +0.000000 0.000000 0.000000 srgb +n 69.650000 26.540000 m 69.650000 30.640000 l 75.600000 30.640000 l 75.600000 26.540000 l cp s +0 slc +0 slj +[] 0 sd +n 70.245000 26.540000 m 70.245000 30.640000 l s +0 slc +0 slj +[] 0 sd +n 69.650000 26.950000 m 75.600000 26.950000 l s +gsave 71.292500 28.595000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 2112 lineto +2624 2112 lineto +2624 3520 lineto +3136 3520 lineto +3136 0 lineto +2624 0 lineto +2624 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 71.774546 28.595000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 72.266580 28.595000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +2496 3520 lineto +2496 3136 lineto +960 3136 lineto +960 2112 lineto +2368 2112 lineto +2368 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 72.633731 28.595000 translate 0.035278 -0.035278 scale +start_ol +512 3520 moveto +2432 3520 lineto +2432 3136 lineto +1024 3136 lineto +1024 2300 lineto +1126 2334 1228 2351 conicto +1330 2368 1432 2368 conicto +2011 2368 2349 2040 conicto +2688 1712 2688 1152 conicto +2688 575 2330 255 conicto +1972 -64 1321 -64 conicto +1097 -64 864 -32 conicto +632 0 384 64 conicto +384 576 lineto +595 447 820 383 conicto +1045 320 1296 320 conicto +1702 320 1939 544 conicto +2176 768 2176 1152 conicto +2176 1536 1936 1760 conicto +1697 1984 1286 1984 conicto +1094 1984 902 1937 conicto +711 1891 512 1792 conicto +512 3520 lineto +end_ol grestore +gsave 73.040850 28.595000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 73.243157 28.595000 translate 0.035278 -0.035278 scale +start_ol +2624 2688 moveto +2624 0 lineto +2176 0 lineto +2176 2368 lineto +960 2368 lineto +960 0 lineto +512 0 lineto +512 2368 lineto +64 2368 lineto +64 2688 lineto +512 2688 lineto +512 2871 lineto +512 3301 718 3506 conicto +924 3712 1351 3712 conicto +1792 3712 lineto +1792 3328 lineto +1376 3328 lineto +1142 3328 1051 3237 conicto +960 3147 960 2912 conicto +960 2688 lineto +2624 2688 lineto +2176 3712 moveto +2624 3712 lineto +2624 3136 lineto +2176 3136 lineto +2176 3712 lineto +end_ol grestore +gsave 73.645274 28.595000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 73.822603 28.595000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 74.217229 28.595000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 71.133750 29.395000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 71.383517 29.395000 translate 0.035278 -0.035278 scale +start_ol +512 3520 moveto +2432 3520 lineto +2432 3136 lineto +1024 3136 lineto +1024 2300 lineto +1126 2334 1228 2351 conicto +1330 2368 1432 2368 conicto +2011 2368 2349 2040 conicto +2688 1712 2688 1152 conicto +2688 575 2330 255 conicto +1972 -64 1321 -64 conicto +1097 -64 864 -32 conicto +632 0 384 64 conicto +384 576 lineto +595 447 820 383 conicto +1045 320 1296 320 conicto +1702 320 1939 544 conicto +2176 768 2176 1152 conicto +2176 1536 1936 1760 conicto +1697 1984 1286 1984 conicto +1094 1984 902 1937 conicto +711 1891 512 1792 conicto +512 3520 lineto +end_ol grestore +gsave 71.790636 29.395000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 72.282670 29.395000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 72.484977 29.395000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 72.877106 29.395000 translate 0.035278 -0.035278 scale +start_ol +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +896 2304 moveto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +end_ol grestore +gsave 73.284225 29.395000 translate 0.035278 -0.035278 scale +start_ol +448 2688 moveto +896 2688 lineto +896 -41 lineto +896 -559 695 -791 conicto +494 -1024 47 -1024 conicto +-128 -1024 lineto +-128 -640 lineto +28 -640 lineto +272 -640 360 -524 conicto +448 -408 448 -41 conicto +448 2688 lineto +448 3712 moveto +896 3712 lineto +896 3136 lineto +448 3136 lineto +448 3712 lineto +end_ol grestore +gsave 73.461554 29.395000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 73.856180 29.395000 translate 0.035278 -0.035278 scale +start_ol +2368 2560 moveto +2368 2176 lineto +2180 2272 1991 2320 conicto +1803 2368 1611 2368 conicto +1180 2368 942 2099 conicto +704 1830 704 1344 conicto +704 858 942 589 conicto +1180 320 1611 320 conicto +1803 320 1991 368 conicto +2180 416 2368 512 conicto +2368 128 lineto +2184 32 1987 -16 conicto +1791 -64 1569 -64 conicto +966 -64 611 316 conicto +256 697 256 1344 conicto +256 2000 615 2376 conicto +974 2752 1598 2752 conicto +1801 2752 1994 2704 conicto +2187 2656 2368 2560 conicto +end_ol grestore +gsave 74.208349 29.395000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 74.458116 29.395000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 35.225000 28.572500 m 37.575000 28.572500 l 37.575000 28.260000 l 39.925000 28.885000 l 37.575000 29.510000 l 37.575000 29.197500 l 35.225000 29.197500 l ef +0.000000 0.000000 0.000000 srgb +n 35.225000 28.572500 m 37.575000 28.572500 l 37.575000 28.260000 l 39.925000 28.885000 l 37.575000 29.510000 l 37.575000 29.197500 l 35.225000 29.197500 l cp s +0 slc +0 slj +[] 0 sd +n 35.225000 28.572500 m 37.575000 28.572500 l 37.575000 28.260000 l 39.925000 28.885000 l 37.575000 29.510000 l 37.575000 29.197500 l 35.225000 29.197500 l cp s +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 50.800000 28.417500 m 53.150000 28.417500 l 53.150000 28.105000 l 55.500000 28.730000 l 53.150000 29.355000 l 53.150000 29.042500 l 50.800000 29.042500 l ef +0.000000 0.000000 0.000000 srgb +n 50.800000 28.417500 m 53.150000 28.417500 l 53.150000 28.105000 l 55.500000 28.730000 l 53.150000 29.355000 l 53.150000 29.042500 l 50.800000 29.042500 l cp s +0 slc +0 slj +[] 0 sd +n 50.800000 28.417500 m 53.150000 28.417500 l 53.150000 28.105000 l 55.500000 28.730000 l 53.150000 29.355000 l 53.150000 29.042500 l 50.800000 29.042500 l cp s +1.000000 1.000000 1.000000 srgb +n 40.550000 26.950000 m 40.550000 30.850000 l 48.650000 30.850000 l 48.650000 26.950000 l f +0.100000 slw +[] 0 sd +[] 0 sd +0 slj +0.000000 0.000000 0.000000 srgb +n 40.550000 26.950000 m 40.550000 30.850000 l 48.650000 30.850000 l 48.650000 26.950000 l cp s +gsave 41.807500 28.295000 translate 0.035278 -0.035278 scale +start_ol +2140 1632 moveto +2295 1581 2442 1413 conicto +2590 1245 2738 952 conicto +3200 0 lineto +2709 0 lineto +2250 893 lineto +2077 1243 1915 1357 conicto +1753 1472 1473 1472 conicto +960 1472 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +1552 3520 lineto +2157 3520 2454 3265 conicto +2752 3010 2752 2496 conicto +2752 2161 2595 1939 conicto +2439 1718 2140 1632 conicto +960 3136 moveto +960 1856 lineto +1552 1856 lineto +1893 1856 2066 2019 conicto +2240 2182 2240 2498 conicto +2240 2815 2066 2975 conicto +1893 3136 1552 3136 conicto +960 3136 lineto +end_ol grestore +gsave 42.224607 28.295000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 42.619233 28.295000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 43.011362 28.295000 translate 0.035278 -0.035278 scale +start_ol +2176 2304 moveto +2176 3712 lineto +2624 3712 lineto +2624 0 lineto +2176 0 lineto +2176 384 lineto +2040 157 1832 46 conicto +1624 -64 1332 -64 conicto +855 -64 555 324 conicto +256 712 256 1344 conicto +256 1976 555 2364 conicto +855 2752 1332 2752 conicto +1624 2752 1832 2641 conicto +2040 2531 2176 2304 conicto +704 1344 moveto +704 865 900 592 conicto +1096 320 1439 320 conicto +1782 320 1979 592 conicto +2176 865 2176 1344 conicto +2176 1823 1979 2095 conicto +1782 2368 1439 2368 conicto +1096 2368 900 2095 conicto +704 1823 704 1344 conicto +end_ol grestore +gsave 43.418482 28.295000 translate 0.035278 -0.035278 scale +start_ol +448 2688 moveto +896 2688 lineto +896 0 lineto +448 0 lineto +448 2688 lineto +448 3712 moveto +896 3712 lineto +896 3136 lineto +448 3136 lineto +448 3712 lineto +end_ol grestore +gsave 43.595810 28.295000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 44.000425 28.295000 translate 0.035278 -0.035278 scale +start_ol +2176 1375 moveto +2176 1848 1982 2108 conicto +1789 2368 1439 2368 conicto +1091 2368 897 2108 conicto +704 1848 704 1375 conicto +704 904 897 644 conicto +1091 384 1439 384 conicto +1789 384 1982 644 conicto +2176 904 2176 1375 conicto +2624 347 moveto +2624 -347 2323 -685 conicto +2023 -1024 1404 -1024 conicto +1174 -1024 971 -992 conicto +768 -961 576 -896 conicto +576 -448 lineto +766 -546 951 -593 conicto +1137 -640 1329 -640 conicto +1754 -640 1965 -415 conicto +2176 -190 2176 264 conicto +2176 448 lineto +2042 223 1833 111 conicto +1624 0 1332 0 conicto +848 0 552 376 conicto +256 753 256 1375 conicto +256 1999 552 2375 conicto +848 2752 1332 2752 conicto +1624 2752 1833 2640 conicto +2042 2529 2176 2304 conicto +2176 2688 lineto +2624 2688 lineto +2624 347 lineto +end_ol grestore +gsave 44.407544 28.295000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 44.609851 28.295000 translate 0.035278 -0.035278 scale +start_ol +2624 3392 moveto +2624 2944 lineto +2352 3073 2111 3136 conicto +1870 3200 1645 3200 conicto +1255 3200 1043 3052 conicto +832 2904 832 2631 conicto +832 2402 969 2285 conicto +1107 2169 1491 2097 conicto +1773 2039 lineto +2308 1937 2562 1678 conicto +2816 1420 2816 986 conicto +2816 469 2468 202 conicto +2121 -64 1450 -64 conicto +1197 -64 911 0 conicto +626 65 320 192 conicto +320 704 lineto +613 513 893 416 conicto +1174 320 1445 320 conicto +1857 320 2080 485 conicto +2304 650 2304 955 conicto +2304 1221 2148 1371 conicto +1992 1522 1636 1597 conicto +1352 1652 lineto +807 1756 563 1978 conicto +320 2201 320 2597 conicto +320 3056 654 3320 conicto +989 3584 1576 3584 conicto +1828 3584 2089 3536 conicto +2351 3488 2624 3392 conicto +end_ol grestore +gsave 45.016970 28.295000 translate 0.035278 -0.035278 scale +start_ol +1890 3200 moveto +1375 3200 1071 2813 conicto +768 2426 768 1759 conicto +768 1094 1071 707 conicto +1375 320 1890 320 conicto +2406 320 2707 707 conicto +3008 1094 3008 1759 conicto +3008 2426 2707 2813 conicto +2406 3200 1890 3200 conicto +2553 67 moveto +3181 -640 lineto +2607 -640 lineto +2087 -55 lineto +2009 -59 1968 -61 conicto +1927 -64 1889 -64 conicto +1146 -64 701 431 conicto +256 927 256 1759 conicto +256 2593 701 3088 conicto +1146 3584 1890 3584 conicto +2632 3584 3076 3088 conicto +3520 2592 3520 1758 conicto +3520 1146 3271 710 conicto +3023 274 2553 67 conicto +end_ol grestore +gsave 45.521498 28.295000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 384 lineto +2688 384 lineto +2688 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 45.878660 28.295000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 46.080967 28.295000 translate 0.035278 -0.035278 scale +start_ol +2624 2688 moveto +2624 0 lineto +2176 0 lineto +2176 2368 lineto +960 2368 lineto +960 0 lineto +512 0 lineto +512 2368 lineto +64 2368 lineto +64 2688 lineto +512 2688 lineto +512 2871 lineto +512 3301 718 3506 conicto +924 3712 1351 3712 conicto +1792 3712 lineto +1792 3328 lineto +1376 3328 lineto +1142 3328 1051 3237 conicto +960 3147 960 2912 conicto +960 2688 lineto +2624 2688 lineto +2176 3712 moveto +2624 3712 lineto +2624 3136 lineto +2176 3136 lineto +2176 3712 lineto +end_ol grestore +gsave 46.483084 28.295000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 46.660413 28.295000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 47.055039 28.295000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 41.507500 29.095000 translate 0.035278 -0.035278 scale +start_ol +448 2688 moveto +896 2688 lineto +896 0 lineto +448 0 lineto +448 2688 lineto +448 3712 moveto +896 3712 lineto +896 3136 lineto +448 3136 lineto +448 3712 lineto +end_ol grestore +gsave 41.684829 29.095000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 42.089443 29.095000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 42.339209 29.095000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 42.731338 29.095000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 42.933646 29.095000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 1792 lineto +1552 1792 lineto +1881 1792 2060 1967 conicto +2240 2142 2240 2465 conicto +2240 2786 2060 2961 conicto +1881 3136 1552 3136 conicto +960 3136 lineto +448 3520 moveto +1552 3520 lineto +2145 3520 2448 3251 conicto +2752 2983 2752 2465 conicto +2752 1943 2448 1675 conicto +2145 1408 1552 1408 conicto +960 1408 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 43.318284 29.095000 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 43.697928 29.095000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 43.947694 29.095000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 44.352308 29.095000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 44.744437 29.095000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 45.149051 29.095000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 45.351358 29.095000 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 45.758478 29.095000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 46.150607 29.095000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 46.555221 29.095000 translate 0.035278 -0.035278 scale +start_ol +2176 2304 moveto +2176 3712 lineto +2624 3712 lineto +2624 0 lineto +2176 0 lineto +2176 384 lineto +2040 157 1832 46 conicto +1624 -64 1332 -64 conicto +855 -64 555 324 conicto +256 712 256 1344 conicto +256 1976 555 2364 conicto +855 2752 1332 2752 conicto +1624 2752 1832 2641 conicto +2040 2531 2176 2304 conicto +704 1344 moveto +704 865 900 592 conicto +1096 320 1439 320 conicto +1782 320 1979 592 conicto +2176 865 2176 1344 conicto +2176 1823 1979 2095 conicto +1782 2368 1439 2368 conicto +1096 2368 900 2095 conicto +704 1823 704 1344 conicto +end_ol grestore +gsave 46.962340 29.095000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 47.354469 29.095000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 42.590000 29.895000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 42.839767 29.895000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 43.331801 29.895000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 43.723930 29.895000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 43.973696 29.895000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 44.365825 29.895000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +2496 3520 lineto +2496 3136 lineto +960 3136 lineto +960 2112 lineto +2368 2112 lineto +2368 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 44.685525 29.895000 translate 0.035278 -0.035278 scale +start_ol +1984 2304 moveto +1912 2337 1828 2352 conicto +1744 2368 1642 2368 conicto +1282 2368 1089 2127 conicto +896 1887 896 1437 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1030 2531 1245 2641 conicto +1460 2752 1767 2752 conicto +1811 2752 1864 2752 conicto +1917 2752 1982 2752 conicto +1984 2304 lineto +end_ol grestore +gsave 44.947776 29.895000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 45.339906 29.895000 translate 0.035278 -0.035278 scale +start_ol +2549 2204 moveto +2714 2485 2942 2618 conicto +3171 2752 3482 2752 conicto +3899 2752 4125 2464 conicto +4352 2177 4352 1646 conicto +4352 0 lineto +3904 0 lineto +3904 1632 lineto +3904 2006 3769 2187 conicto +3634 2368 3356 2368 conicto +3017 2368 2820 2146 conicto +2624 1925 2624 1542 conicto +2624 0 lineto +2176 0 lineto +2176 1632 lineto +2176 2008 2041 2188 conicto +1906 2368 1624 2368 conicto +1289 2368 1092 2145 conicto +896 1922 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1046 2534 1255 2643 conicto +1465 2752 1753 2752 conicto +2044 2752 2247 2611 conicto +2451 2471 2549 2204 conicto +end_ol grestore +gsave 45.961817 29.895000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 46.356443 29.895000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +gsave 29.325000 31.840000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 29.574767 31.840000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 2112 lineto +2624 2112 lineto +2624 3520 lineto +3136 3520 lineto +3136 0 lineto +2624 0 lineto +2624 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.056813 31.840000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.548847 31.840000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 31.040881 31.840000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +gsave 71.700000 31.685000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 71.949767 31.685000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 2112 lineto +2624 2112 lineto +2624 3520 lineto +3136 3520 lineto +3136 0 lineto +2624 0 lineto +2624 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 72.431813 31.685000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 72.923847 31.685000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 73.415881 31.685000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +gsave 43.525000 31.730000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 43.774767 31.730000 translate 0.035278 -0.035278 scale +start_ol +2140 1632 moveto +2295 1581 2442 1413 conicto +2590 1245 2738 952 conicto +3200 0 lineto +2709 0 lineto +2250 893 lineto +2077 1243 1915 1357 conicto +1753 1472 1473 1472 conicto +960 1472 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +1552 3520 lineto +2157 3520 2454 3265 conicto +2752 3010 2752 2496 conicto +2752 2161 2595 1939 conicto +2439 1718 2140 1632 conicto +960 3136 moveto +960 1856 lineto +1552 1856 lineto +1893 1856 2066 2019 conicto +2240 2182 2240 2498 conicto +2240 2815 2066 2975 conicto +1893 3136 1552 3136 conicto +960 3136 lineto +end_ol grestore +gsave 44.194371 31.730000 translate 0.035278 -0.035278 scale +start_ol +1696 3047 moveto +1042 1280 lineto +2352 1280 lineto +1696 3047 lineto +1424 3520 moveto +1970 3520 lineto +3328 0 lineto +2827 0 lineto +2502 896 lineto +897 896 lineto +572 0 lineto +64 0 lineto +1424 3520 lineto +end_ol grestore +gsave 44.631463 31.730000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +1202 3520 lineto +2079 1148 lineto +2961 3520 lineto +3712 3520 lineto +3712 0 lineto +3200 0 lineto +3200 3095 lineto +2314 704 lineto +1846 704 lineto +960 3095 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 45.183441 31.730000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 27.325000 33.145000 m 27.325000 37.245000 l 33.275000 37.245000 l 33.275000 33.145000 l f +0.000000 0.000000 0.000000 srgb +n 27.325000 33.145000 m 27.325000 37.245000 l 33.275000 37.245000 l 33.275000 33.145000 l cp s +0 slc +0 slj +[] 0 sd +n 27.920000 33.145000 m 27.920000 37.245000 l s +0 slc +0 slj +[] 0 sd +n 27.325000 33.555000 m 33.275000 33.555000 l s +gsave 28.967500 35.200000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 2112 lineto +2624 2112 lineto +2624 3520 lineto +3136 3520 lineto +3136 0 lineto +2624 0 lineto +2624 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 29.449546 35.200000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 29.941580 35.200000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +2496 3520 lineto +2496 3136 lineto +960 3136 lineto +960 2112 lineto +2368 2112 lineto +2368 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.308731 35.200000 translate 0.035278 -0.035278 scale +start_ol +512 3520 moveto +2432 3520 lineto +2432 3136 lineto +1024 3136 lineto +1024 2300 lineto +1126 2334 1228 2351 conicto +1330 2368 1432 2368 conicto +2011 2368 2349 2040 conicto +2688 1712 2688 1152 conicto +2688 575 2330 255 conicto +1972 -64 1321 -64 conicto +1097 -64 864 -32 conicto +632 0 384 64 conicto +384 576 lineto +595 447 820 383 conicto +1045 320 1296 320 conicto +1702 320 1939 544 conicto +2176 768 2176 1152 conicto +2176 1536 1936 1760 conicto +1697 1984 1286 1984 conicto +1094 1984 902 1937 conicto +711 1891 512 1792 conicto +512 3520 lineto +end_ol grestore +gsave 30.715850 35.200000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 30.918157 35.200000 translate 0.035278 -0.035278 scale +start_ol +2624 2688 moveto +2624 0 lineto +2176 0 lineto +2176 2368 lineto +960 2368 lineto +960 0 lineto +512 0 lineto +512 2368 lineto +64 2368 lineto +64 2688 lineto +512 2688 lineto +512 2871 lineto +512 3301 718 3506 conicto +924 3712 1351 3712 conicto +1792 3712 lineto +1792 3328 lineto +1376 3328 lineto +1142 3328 1051 3237 conicto +960 3147 960 2912 conicto +960 2688 lineto +2624 2688 lineto +2176 3712 moveto +2624 3712 lineto +2624 3136 lineto +2176 3136 lineto +2176 3712 lineto +end_ol grestore +gsave 31.320274 35.200000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 31.497603 35.200000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 31.892229 35.200000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 28.808750 36.000000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 29.058517 36.000000 translate 0.035278 -0.035278 scale +start_ol +512 3520 moveto +2432 3520 lineto +2432 3136 lineto +1024 3136 lineto +1024 2300 lineto +1126 2334 1228 2351 conicto +1330 2368 1432 2368 conicto +2011 2368 2349 2040 conicto +2688 1712 2688 1152 conicto +2688 575 2330 255 conicto +1972 -64 1321 -64 conicto +1097 -64 864 -32 conicto +632 0 384 64 conicto +384 576 lineto +595 447 820 383 conicto +1045 320 1296 320 conicto +1702 320 1939 544 conicto +2176 768 2176 1152 conicto +2176 1536 1936 1760 conicto +1697 1984 1286 1984 conicto +1094 1984 902 1937 conicto +711 1891 512 1792 conicto +512 3520 lineto +end_ol grestore +gsave 29.465636 36.000000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 29.957670 36.000000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 30.159977 36.000000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 30.552106 36.000000 translate 0.035278 -0.035278 scale +start_ol +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +896 2304 moveto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +end_ol grestore +gsave 30.959225 36.000000 translate 0.035278 -0.035278 scale +start_ol +448 2688 moveto +896 2688 lineto +896 -41 lineto +896 -559 695 -791 conicto +494 -1024 47 -1024 conicto +-128 -1024 lineto +-128 -640 lineto +28 -640 lineto +272 -640 360 -524 conicto +448 -408 448 -41 conicto +448 2688 lineto +448 3712 moveto +896 3712 lineto +896 3136 lineto +448 3136 lineto +448 3712 lineto +end_ol grestore +gsave 31.136554 36.000000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 31.531180 36.000000 translate 0.035278 -0.035278 scale +start_ol +2368 2560 moveto +2368 2176 lineto +2180 2272 1991 2320 conicto +1803 2368 1611 2368 conicto +1180 2368 942 2099 conicto +704 1830 704 1344 conicto +704 858 942 589 conicto +1180 320 1611 320 conicto +1803 320 1991 368 conicto +2180 416 2368 512 conicto +2368 128 lineto +2184 32 1987 -16 conicto +1791 -64 1569 -64 conicto +966 -64 611 316 conicto +256 697 256 1344 conicto +256 2000 615 2376 conicto +974 2752 1598 2752 conicto +1801 2752 1994 2704 conicto +2187 2656 2368 2560 conicto +end_ol grestore +gsave 31.883349 36.000000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 32.133116 36.000000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 35.325000 34.667500 m 37.675000 34.667500 l 37.675000 34.355000 l 40.025000 34.980000 l 37.675000 35.605000 l 37.675000 35.292500 l 35.325000 35.292500 l ef +0.000000 0.000000 0.000000 srgb +n 35.325000 34.667500 m 37.675000 34.667500 l 37.675000 34.355000 l 40.025000 34.980000 l 37.675000 35.605000 l 37.675000 35.292500 l 35.325000 35.292500 l cp s +0 slc +0 slj +[] 0 sd +n 35.325000 34.667500 m 37.675000 34.667500 l 37.675000 34.355000 l 40.025000 34.980000 l 37.675000 35.605000 l 37.675000 35.292500 l 35.325000 35.292500 l cp s +1.000000 1.000000 1.000000 srgb +n 40.600000 32.845000 m 40.600000 37.145000 l 48.386250 37.145000 l 48.386250 32.845000 l f +0.100000 slw +[] 0 sd +[] 0 sd +0 slj +0.000000 0.000000 0.000000 srgb +n 40.600000 32.845000 m 40.600000 37.145000 l 48.386250 37.145000 l 48.386250 32.845000 l cp s +gsave 41.460625 34.390000 translate 0.035278 -0.035278 scale +start_ol +2140 1632 moveto +2295 1581 2442 1413 conicto +2590 1245 2738 952 conicto +3200 0 lineto +2709 0 lineto +2250 893 lineto +2077 1243 1915 1357 conicto +1753 1472 1473 1472 conicto +960 1472 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +1552 3520 lineto +2157 3520 2454 3265 conicto +2752 3010 2752 2496 conicto +2752 2161 2595 1939 conicto +2439 1718 2140 1632 conicto +960 3136 moveto +960 1856 lineto +1552 1856 lineto +1893 1856 2066 2019 conicto +2240 2182 2240 2498 conicto +2240 2815 2066 2975 conicto +1893 3136 1552 3136 conicto +960 3136 lineto +end_ol grestore +gsave 41.877732 34.390000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 42.272358 34.390000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 42.664487 34.390000 translate 0.035278 -0.035278 scale +start_ol +2176 2304 moveto +2176 3712 lineto +2624 3712 lineto +2624 0 lineto +2176 0 lineto +2176 384 lineto +2040 157 1832 46 conicto +1624 -64 1332 -64 conicto +855 -64 555 324 conicto +256 712 256 1344 conicto +256 1976 555 2364 conicto +855 2752 1332 2752 conicto +1624 2752 1832 2641 conicto +2040 2531 2176 2304 conicto +704 1344 moveto +704 865 900 592 conicto +1096 320 1439 320 conicto +1782 320 1979 592 conicto +2176 865 2176 1344 conicto +2176 1823 1979 2095 conicto +1782 2368 1439 2368 conicto +1096 2368 900 2095 conicto +704 1823 704 1344 conicto +end_ol grestore +gsave 43.071607 34.390000 translate 0.035278 -0.035278 scale +start_ol +448 2688 moveto +896 2688 lineto +896 0 lineto +448 0 lineto +448 2688 lineto +448 3712 moveto +896 3712 lineto +896 3136 lineto +448 3136 lineto +448 3712 lineto +end_ol grestore +gsave 43.248935 34.390000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 43.653550 34.390000 translate 0.035278 -0.035278 scale +start_ol +2176 1375 moveto +2176 1848 1982 2108 conicto +1789 2368 1439 2368 conicto +1091 2368 897 2108 conicto +704 1848 704 1375 conicto +704 904 897 644 conicto +1091 384 1439 384 conicto +1789 384 1982 644 conicto +2176 904 2176 1375 conicto +2624 347 moveto +2624 -347 2323 -685 conicto +2023 -1024 1404 -1024 conicto +1174 -1024 971 -992 conicto +768 -961 576 -896 conicto +576 -448 lineto +766 -546 951 -593 conicto +1137 -640 1329 -640 conicto +1754 -640 1965 -415 conicto +2176 -190 2176 264 conicto +2176 448 lineto +2042 223 1833 111 conicto +1624 0 1332 0 conicto +848 0 552 376 conicto +256 753 256 1375 conicto +256 1999 552 2375 conicto +848 2752 1332 2752 conicto +1624 2752 1833 2640 conicto +2042 2529 2176 2304 conicto +2176 2688 lineto +2624 2688 lineto +2624 347 lineto +end_ol grestore +gsave 44.060669 34.390000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 44.262976 34.390000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 2112 lineto +2624 2112 lineto +2624 3520 lineto +3136 3520 lineto +3136 0 lineto +2624 0 lineto +2624 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 44.745022 34.390000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 45.237056 34.390000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +2496 3520 lineto +2496 3136 lineto +960 3136 lineto +960 2112 lineto +2368 2112 lineto +2368 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 45.604207 34.390000 translate 0.035278 -0.035278 scale +start_ol +512 3520 moveto +2432 3520 lineto +2432 3136 lineto +1024 3136 lineto +1024 2300 lineto +1126 2334 1228 2351 conicto +1330 2368 1432 2368 conicto +2011 2368 2349 2040 conicto +2688 1712 2688 1152 conicto +2688 575 2330 255 conicto +1972 -64 1321 -64 conicto +1097 -64 864 -32 conicto +632 0 384 64 conicto +384 576 lineto +595 447 820 383 conicto +1045 320 1296 320 conicto +1702 320 1939 544 conicto +2176 768 2176 1152 conicto +2176 1536 1936 1760 conicto +1697 1984 1286 1984 conicto +1094 1984 902 1937 conicto +711 1891 512 1792 conicto +512 3520 lineto +end_ol grestore +gsave 46.011326 34.390000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 46.213633 34.390000 translate 0.035278 -0.035278 scale +start_ol +2624 2688 moveto +2624 0 lineto +2176 0 lineto +2176 2368 lineto +960 2368 lineto +960 0 lineto +512 0 lineto +512 2368 lineto +64 2368 lineto +64 2688 lineto +512 2688 lineto +512 2871 lineto +512 3301 718 3506 conicto +924 3712 1351 3712 conicto +1792 3712 lineto +1792 3328 lineto +1376 3328 lineto +1142 3328 1051 3237 conicto +960 3147 960 2912 conicto +960 2688 lineto +2624 2688 lineto +2176 3712 moveto +2624 3712 lineto +2624 3136 lineto +2176 3136 lineto +2176 3712 lineto +end_ol grestore +gsave 46.615750 34.390000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 46.793079 34.390000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 47.187705 34.390000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 41.400625 35.190000 translate 0.035278 -0.035278 scale +start_ol +448 2688 moveto +896 2688 lineto +896 0 lineto +448 0 lineto +448 2688 lineto +448 3712 moveto +896 3712 lineto +896 3136 lineto +448 3136 lineto +448 3712 lineto +end_ol grestore +gsave 41.577954 35.190000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 41.982568 35.190000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 42.232334 35.190000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 42.624463 35.190000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 42.826771 35.190000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 1792 lineto +1552 1792 lineto +1881 1792 2060 1967 conicto +2240 2142 2240 2465 conicto +2240 2786 2060 2961 conicto +1881 3136 1552 3136 conicto +960 3136 lineto +448 3520 moveto +1552 3520 lineto +2145 3520 2448 3251 conicto +2752 2983 2752 2465 conicto +2752 1943 2448 1675 conicto +2145 1408 1552 1408 conicto +960 1408 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 43.211409 35.190000 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 43.591053 35.190000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 43.840819 35.190000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 44.245433 35.190000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 44.637562 35.190000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 45.042176 35.190000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 45.244483 35.190000 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 45.651603 35.190000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 46.043732 35.190000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 46.448346 35.190000 translate 0.035278 -0.035278 scale +start_ol +2176 2304 moveto +2176 3712 lineto +2624 3712 lineto +2624 0 lineto +2176 0 lineto +2176 384 lineto +2040 157 1832 46 conicto +1624 -64 1332 -64 conicto +855 -64 555 324 conicto +256 712 256 1344 conicto +256 1976 555 2364 conicto +855 2752 1332 2752 conicto +1624 2752 1832 2641 conicto +2040 2531 2176 2304 conicto +704 1344 moveto +704 865 900 592 conicto +1096 320 1439 320 conicto +1782 320 1979 592 conicto +2176 865 2176 1344 conicto +2176 1823 1979 2095 conicto +1782 2368 1439 2368 conicto +1096 2368 900 2095 conicto +704 1823 704 1344 conicto +end_ol grestore +gsave 46.855465 35.190000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 47.247594 35.190000 translate 0.035278 -0.035278 scale +start_ol +2112 2560 moveto +2112 2176 lineto +1932 2272 1738 2320 conicto +1544 2368 1336 2368 conicto +1020 2368 862 2269 conicto +704 2170 704 1972 conicto +704 1821 814 1735 conicto +925 1649 1260 1571 conicto +1403 1538 lineto +1857 1438 2048 1255 conicto +2240 1072 2240 744 conicto +2240 371 1954 153 conicto +1668 -64 1167 -64 conicto +958 -64 732 -16 conicto +506 32 256 128 conicto +256 576 lineto +491 448 719 384 conicto +947 320 1170 320 conicto +1470 320 1631 425 conicto +1792 531 1792 722 conicto +1792 900 1678 994 conicto +1564 1089 1177 1177 conicto +1032 1212 lineto +621 1298 438 1476 conicto +256 1654 256 1964 conicto +256 2341 520 2546 conicto +784 2752 1269 2752 conicto +1509 2752 1721 2704 conicto +1933 2656 2112 2560 conicto +end_ol grestore +gsave 42.064375 35.990000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 42.314142 35.990000 translate 0.035278 -0.035278 scale +start_ol +1897 1936 moveto +2269 1862 2478 1627 conicto +2688 1392 2688 1046 conicto +2688 516 2331 226 conicto +1975 -64 1319 -64 conicto +1099 -64 865 -16 conicto +632 32 384 128 conicto +384 576 lineto +576 449 805 384 conicto +1035 320 1285 320 conicto +1720 320 1948 504 conicto +2176 688 2176 1039 conicto +2176 1363 1967 1545 conicto +1759 1728 1388 1728 conicto +1024 1728 lineto +1024 2112 lineto +1405 2112 lineto +1717 2112 1882 2250 conicto +2048 2388 2048 2648 conicto +2048 2915 1877 3057 conicto +1706 3200 1388 3200 conicto +1200 3200 985 3153 conicto +770 3107 512 3008 conicto +512 3456 lineto +773 3520 1001 3552 conicto +1229 3584 1431 3584 conicto +1952 3584 2256 3343 conicto +2560 3102 2560 2692 conicto +2560 2406 2387 2209 conicto +2215 2012 1897 1936 conicto +end_ol grestore +gsave 42.721261 35.990000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 43.213295 35.990000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 43.415602 35.990000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 43.907636 35.990000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 44.299765 35.990000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 44.549532 35.990000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 44.941661 35.990000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 1792 lineto +1552 1792 lineto +1881 1792 2060 1967 conicto +2240 2142 2240 2465 conicto +2240 2786 2060 2961 conicto +1881 3136 1552 3136 conicto +960 3136 lineto +448 3520 moveto +1552 3520 lineto +2145 3520 2448 3251 conicto +2752 2983 2752 2465 conicto +2752 1943 2448 1675 conicto +2145 1408 1552 1408 conicto +960 1408 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 45.298824 35.990000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 45.690953 35.990000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 46.095567 35.990000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 46.490193 35.990000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 46.667521 35.990000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +gsave 43.825000 38.090000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 44.074767 38.090000 translate 0.035278 -0.035278 scale +start_ol +2140 1632 moveto +2295 1581 2442 1413 conicto +2590 1245 2738 952 conicto +3200 0 lineto +2709 0 lineto +2250 893 lineto +2077 1243 1915 1357 conicto +1753 1472 1473 1472 conicto +960 1472 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +1552 3520 lineto +2157 3520 2454 3265 conicto +2752 3010 2752 2496 conicto +2752 2161 2595 1939 conicto +2439 1718 2140 1632 conicto +960 3136 moveto +960 1856 lineto +1552 1856 lineto +1893 1856 2066 2019 conicto +2240 2182 2240 2498 conicto +2240 2815 2066 2975 conicto +1893 3136 1552 3136 conicto +960 3136 lineto +end_ol grestore +gsave 44.494371 38.090000 translate 0.035278 -0.035278 scale +start_ol +1696 3047 moveto +1042 1280 lineto +2352 1280 lineto +1696 3047 lineto +1424 3520 moveto +1970 3520 lineto +3328 0 lineto +2827 0 lineto +2502 896 lineto +897 896 lineto +572 0 lineto +64 0 lineto +1424 3520 lineto +end_ol grestore +gsave 44.931463 38.090000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +1202 3520 lineto +2079 1148 lineto +2961 3520 lineto +3712 3520 lineto +3712 0 lineto +3200 0 lineto +3200 3095 lineto +2314 704 lineto +1846 704 lineto +960 3095 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 45.483441 38.090000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 34.210000 28.345000 m 36.750000 28.345000 l 37.100701 28.345000 37.385000 28.591243 37.385000 28.895000 c 37.385000 29.198757 37.100701 29.445000 36.750000 29.445000 c 34.210000 29.445000 l 33.859299 29.445000 33.575000 29.198757 33.575000 28.895000 c 33.575000 28.591243 33.859299 28.345000 34.210000 28.345000 c ef +0.000000 0.000000 0.000000 srgb +n 34.210000 28.345000 m 36.750000 28.345000 l 37.100701 28.345000 37.385000 28.591243 37.385000 28.895000 c 37.385000 29.198757 37.100701 29.445000 36.750000 29.445000 c 34.210000 29.445000 l 33.859299 29.445000 33.575000 29.198757 33.575000 28.895000 c 33.575000 28.591243 33.859299 28.345000 34.210000 28.345000 c s +gsave 34.360000 29.095000 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 34.767119 29.095000 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 35.146763 29.095000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 35.396530 29.095000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 35.801144 29.095000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 36.193273 29.095000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 49.785000 28.190000 m 52.325000 28.190000 l 52.675701 28.190000 52.960000 28.436243 52.960000 28.740000 c 52.960000 29.043757 52.675701 29.290000 52.325000 29.290000 c 49.785000 29.290000 l 49.434299 29.290000 49.150000 29.043757 49.150000 28.740000 c 49.150000 28.436243 49.434299 28.190000 49.785000 28.190000 c ef +0.000000 0.000000 0.000000 srgb +n 49.785000 28.190000 m 52.325000 28.190000 l 52.675701 28.190000 52.960000 28.436243 52.960000 28.740000 c 52.960000 29.043757 52.675701 29.290000 52.325000 29.290000 c 49.785000 29.290000 l 49.434299 29.290000 49.150000 29.043757 49.150000 28.740000 c 49.150000 28.436243 49.434299 28.190000 49.785000 28.190000 c s +gsave 49.935000 28.940000 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 50.342119 28.940000 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 50.721763 28.940000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 50.971530 28.940000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 51.376144 28.940000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 51.768273 28.940000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 34.310000 34.385000 m 36.850000 34.385000 l 37.200701 34.385000 37.485000 34.631243 37.485000 34.935000 c 37.485000 35.238757 37.200701 35.485000 36.850000 35.485000 c 34.310000 35.485000 l 33.959299 35.485000 33.675000 35.238757 33.675000 34.935000 c 33.675000 34.631243 33.959299 34.385000 34.310000 34.385000 c ef +0.000000 0.000000 0.000000 srgb +n 34.310000 34.385000 m 36.850000 34.385000 l 37.200701 34.385000 37.485000 34.631243 37.485000 34.935000 c 37.485000 35.238757 37.200701 35.485000 36.850000 35.485000 c 34.310000 35.485000 l 33.959299 35.485000 33.675000 35.238757 33.675000 34.935000 c 33.675000 34.631243 33.959299 34.385000 34.310000 34.385000 c s +gsave 34.460000 35.135000 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 34.867119 35.135000 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 35.246763 35.135000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 35.496530 35.135000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 35.901144 35.135000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 36.293273 35.135000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 64.125000 28.472536 m 66.475000 28.472536 l 66.475000 28.160036 l 68.825000 28.785036 l 66.475000 29.410036 l 66.475000 29.097536 l 64.125000 29.097536 l ef +0.000000 0.000000 0.000000 srgb +n 64.125000 28.472536 m 66.475000 28.472536 l 66.475000 28.160036 l 68.825000 28.785036 l 66.475000 29.410036 l 66.475000 29.097536 l 64.125000 29.097536 l cp s +0 slc +0 slj +[] 0 sd +n 64.125000 28.472536 m 66.475000 28.472536 l 66.475000 28.160036 l 68.825000 28.785036 l 66.475000 29.410036 l 66.475000 29.097536 l 64.125000 29.097536 l cp s +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 63.110000 28.245036 m 65.650000 28.245036 l 66.000701 28.245036 66.285000 28.491279 66.285000 28.795036 c 66.285000 29.098793 66.000701 29.345036 65.650000 29.345036 c 63.110000 29.345036 l 62.759299 29.345036 62.475000 29.098793 62.475000 28.795036 c 62.475000 28.491279 62.759299 28.245036 63.110000 28.245036 c ef +0.000000 0.000000 0.000000 srgb +n 63.110000 28.245036 m 65.650000 28.245036 l 66.000701 28.245036 66.285000 28.491279 66.285000 28.795036 c 66.285000 29.098793 66.000701 29.345036 65.650000 29.345036 c 63.110000 29.345036 l 62.759299 29.345036 62.475000 29.098793 62.475000 28.795036 c 62.475000 28.491279 62.759299 28.245036 63.110000 28.245036 c s +gsave 63.260000 28.995036 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 63.667119 28.995036 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 64.046763 28.995036 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 64.296530 28.995036 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 64.701144 28.995036 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 65.093273 28.995036 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +1.000000 1.000000 1.000000 srgb +n 56.316250 26.945000 m 56.316250 30.845000 l 61.906250 30.845000 l 61.906250 26.945000 l f +0.100000 slw +[] 0 sd +[] 0 sd +0 slj +0.000000 0.000000 0.000000 srgb +n 56.316250 26.945000 m 56.316250 30.845000 l 61.906250 30.845000 l 61.906250 26.945000 l cp s +gsave 57.845000 28.290000 translate 0.035278 -0.035278 scale +start_ol +3136 3264 moveto +3136 2752 lineto +2892 2977 2616 3088 conicto +2340 3200 2030 3200 conicto +1418 3200 1093 2829 conicto +768 2459 768 1759 conicto +768 1061 1093 690 conicto +1418 320 2030 320 conicto +2340 320 2616 431 conicto +2892 543 3136 768 conicto +3136 256 lineto +2882 96 2599 16 conicto +2316 -64 2000 -64 conicto +1189 -64 722 424 conicto +256 913 256 1759 conicto +256 2607 722 3095 conicto +1189 3584 2000 3584 conicto +2320 3584 2603 3504 conicto +2887 3424 3136 3264 conicto +end_ol grestore +gsave 58.292080 28.290000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 58.684209 28.290000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 59.088823 28.290000 translate 0.035278 -0.035278 scale +start_ol +128 2688 moveto +586 2688 lineto +1408 432 lineto +2230 2688 lineto +2688 2688 lineto +1702 0 lineto +1114 0 lineto +128 2688 lineto +end_ol grestore +gsave 59.468467 28.290000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 59.863093 28.290000 translate 0.035278 -0.035278 scale +start_ol +1984 2304 moveto +1912 2337 1828 2352 conicto +1744 2368 1642 2368 conicto +1282 2368 1089 2127 conicto +896 1887 896 1437 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1030 2531 1245 2641 conicto +1460 2752 1767 2752 conicto +1811 2752 1864 2752 conicto +1917 2752 1982 2752 conicto +1984 2304 lineto +end_ol grestore +gsave 60.125345 28.290000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 56.866250 29.090000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 57.358284 29.090000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 57.750413 29.090000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 58.000180 29.090000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 58.392309 29.090000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +2496 3520 lineto +2496 3136 lineto +960 3136 lineto +960 2112 lineto +2368 2112 lineto +2368 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 58.712008 29.090000 translate 0.035278 -0.035278 scale +start_ol +1984 2304 moveto +1912 2337 1828 2352 conicto +1744 2368 1642 2368 conicto +1282 2368 1089 2127 conicto +896 1887 896 1437 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1030 2531 1245 2641 conicto +1460 2752 1767 2752 conicto +1811 2752 1864 2752 conicto +1917 2752 1982 2752 conicto +1984 2304 lineto +end_ol grestore +gsave 58.974260 29.090000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 59.366389 29.090000 translate 0.035278 -0.035278 scale +start_ol +2549 2204 moveto +2714 2485 2942 2618 conicto +3171 2752 3482 2752 conicto +3899 2752 4125 2464 conicto +4352 2177 4352 1646 conicto +4352 0 lineto +3904 0 lineto +3904 1632 lineto +3904 2006 3769 2187 conicto +3634 2368 3356 2368 conicto +3017 2368 2820 2146 conicto +2624 1925 2624 1542 conicto +2624 0 lineto +2176 0 lineto +2176 1632 lineto +2176 2008 2041 2188 conicto +1906 2368 1624 2368 conicto +1289 2368 1092 2145 conicto +896 1922 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1046 2534 1255 2643 conicto +1465 2752 1753 2752 conicto +2044 2752 2247 2611 conicto +2451 2471 2549 2204 conicto +end_ol grestore +gsave 59.988300 29.090000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 60.382926 29.090000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 60.585233 29.090000 translate 0.035278 -0.035278 scale +start_ol +256 1536 moveto +1536 1536 lineto +1536 1152 lineto +256 1152 lineto +256 1536 lineto +end_ol grestore +gsave 60.815016 29.090000 translate 0.035278 -0.035278 scale +start_ol +512 2380 moveto +512 2816 lineto +3584 1703 lineto +3584 1305 lineto +512 192 lineto +512 628 lineto +2980 1502 lineto +512 2380 lineto +end_ol grestore +gsave 56.932500 29.890000 translate 0.035278 -0.035278 scale +start_ol +1897 1936 moveto +2269 1862 2478 1627 conicto +2688 1392 2688 1046 conicto +2688 516 2331 226 conicto +1975 -64 1319 -64 conicto +1099 -64 865 -16 conicto +632 32 384 128 conicto +384 576 lineto +576 449 805 384 conicto +1035 320 1285 320 conicto +1720 320 1948 504 conicto +2176 688 2176 1039 conicto +2176 1363 1967 1545 conicto +1759 1728 1388 1728 conicto +1024 1728 lineto +1024 2112 lineto +1405 2112 lineto +1717 2112 1882 2250 conicto +2048 2388 2048 2648 conicto +2048 2915 1877 3057 conicto +1706 3200 1388 3200 conicto +1200 3200 985 3153 conicto +770 3107 512 3008 conicto +512 3456 lineto +773 3520 1001 3552 conicto +1229 3584 1431 3584 conicto +1952 3584 2256 3343 conicto +2560 3102 2560 2692 conicto +2560 2406 2387 2209 conicto +2215 2012 1897 1936 conicto +end_ol grestore +gsave 57.339619 29.890000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 57.831653 29.890000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 58.033960 29.890000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 58.525994 29.890000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 58.918124 29.890000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 59.167890 29.890000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 59.560019 29.890000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 1792 lineto +1552 1792 lineto +1881 1792 2060 1967 conicto +2240 2142 2240 2465 conicto +2240 2786 2060 2961 conicto +1881 3136 1552 3136 conicto +960 3136 lineto +448 3520 moveto +1552 3520 lineto +2145 3520 2448 3251 conicto +2752 2983 2752 2465 conicto +2752 1943 2448 1675 conicto +2145 1408 1552 1408 conicto +960 1408 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 59.917182 29.890000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 60.309311 29.890000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 60.713925 29.890000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 61.108551 29.890000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 57.850000 31.725000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 58.099767 31.725000 translate 0.035278 -0.035278 scale +start_ol +2140 1632 moveto +2295 1581 2442 1413 conicto +2590 1245 2738 952 conicto +3200 0 lineto +2709 0 lineto +2250 893 lineto +2077 1243 1915 1357 conicto +1753 1472 1473 1472 conicto +960 1472 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +1552 3520 lineto +2157 3520 2454 3265 conicto +2752 3010 2752 2496 conicto +2752 2161 2595 1939 conicto +2439 1718 2140 1632 conicto +960 3136 moveto +960 1856 lineto +1552 1856 lineto +1893 1856 2066 2019 conicto +2240 2182 2240 2498 conicto +2240 2815 2066 2975 conicto +1893 3136 1552 3136 conicto +960 3136 lineto +end_ol grestore +gsave 58.519371 31.725000 translate 0.035278 -0.035278 scale +start_ol +1696 3047 moveto +1042 1280 lineto +2352 1280 lineto +1696 3047 lineto +1424 3520 moveto +1970 3520 lineto +3328 0 lineto +2827 0 lineto +2502 896 lineto +897 896 lineto +572 0 lineto +64 0 lineto +1424 3520 lineto +end_ol grestore +gsave 58.956463 31.725000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +1202 3520 lineto +2079 1148 lineto +2961 3520 lineto +3712 3520 lineto +3712 0 lineto +3200 0 lineto +3200 3095 lineto +2314 704 lineto +1846 704 lineto +960 3095 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 59.508441 31.725000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +gsave 29.275000 38.290000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 29.524767 38.290000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +960 3520 lineto +960 2112 lineto +2624 2112 lineto +2624 3520 lineto +3136 3520 lineto +3136 0 lineto +2624 0 lineto +2624 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.006813 38.290000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.498847 38.290000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 30.990881 38.290000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 50.675000 34.267500 m 53.025000 34.267500 l 53.025000 33.955000 l 55.375000 34.580000 l 53.025000 35.205000 l 53.025000 34.892500 l 50.675000 34.892500 l ef +0.000000 0.000000 0.000000 srgb +n 50.675000 34.267500 m 53.025000 34.267500 l 53.025000 33.955000 l 55.375000 34.580000 l 53.025000 35.205000 l 53.025000 34.892500 l 50.675000 34.892500 l cp s +0 slc +0 slj +[] 0 sd +n 50.675000 34.267500 m 53.025000 34.267500 l 53.025000 33.955000 l 55.375000 34.580000 l 53.025000 35.205000 l 53.025000 34.892500 l 50.675000 34.892500 l cp s +0.100000 slw +[] 0 sd +[] 0 sd +0 slc +0 slj +0.100000 slw +0 slc +0 slj +[] 0 sd +1.000000 1.000000 1.000000 srgb +n 49.660000 34.040000 m 52.200000 34.040000 l 52.550701 34.040000 52.835000 34.286243 52.835000 34.590000 c 52.835000 34.893757 52.550701 35.140000 52.200000 35.140000 c 49.660000 35.140000 l 49.309299 35.140000 49.025000 34.893757 49.025000 34.590000 c 49.025000 34.286243 49.309299 34.040000 49.660000 34.040000 c ef +0.000000 0.000000 0.000000 srgb +n 49.660000 34.040000 m 52.200000 34.040000 l 52.550701 34.040000 52.835000 34.286243 52.835000 34.590000 c 52.835000 34.893757 52.550701 35.140000 52.200000 35.140000 c 49.660000 35.140000 l 49.309299 35.140000 49.025000 34.893757 49.025000 34.590000 c 49.025000 34.286243 49.309299 34.040000 49.660000 34.040000 c s +gsave 49.810000 34.790000 translate 0.035278 -0.035278 scale +start_ol +896 384 moveto +896 -1024 lineto +448 -1024 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1032 2531 1240 2641 conicto +1448 2752 1737 2752 conicto +2217 2752 2516 2364 conicto +2816 1976 2816 1344 conicto +2816 712 2516 324 conicto +2217 -64 1737 -64 conicto +1448 -64 1240 46 conicto +1032 157 896 384 conicto +2368 1344 moveto +2368 1823 2171 2095 conicto +1975 2368 1632 2368 conicto +1289 2368 1092 2095 conicto +896 1823 896 1344 conicto +896 865 1092 592 conicto +1289 320 1632 320 conicto +1975 320 2171 592 conicto +2368 865 2368 1344 conicto +end_ol grestore +gsave 50.217119 34.790000 translate 0.035278 -0.035278 scale +start_ol +1542 -238 moveto +1360 -726 1187 -875 conicto +1015 -1024 726 -1024 conicto +384 -1024 lineto +384 -640 lineto +636 -640 lineto +813 -640 910 -555 conicto +1008 -470 1127 -155 conicto +1204 46 lineto +128 2688 lineto +603 2688 lineto +1418 586 lineto +2234 2688 lineto +2688 2688 lineto +1542 -238 lineto +end_ol grestore +gsave 50.596763 34.790000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 50.846530 34.790000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 3712 lineto +896 3712 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 51.251144 34.790000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 51.643273 34.790000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +1.000000 1.000000 1.000000 srgb +n 56.191250 32.795000 m 56.191250 36.695000 l 61.781250 36.695000 l 61.781250 32.795000 l f +0.100000 slw +[] 0 sd +[] 0 sd +0 slj +0.000000 0.000000 0.000000 srgb +n 56.191250 32.795000 m 56.191250 36.695000 l 61.781250 36.695000 l 61.781250 32.795000 l cp s +gsave 57.720000 34.140000 translate 0.035278 -0.035278 scale +start_ol +3136 3264 moveto +3136 2752 lineto +2892 2977 2616 3088 conicto +2340 3200 2030 3200 conicto +1418 3200 1093 2829 conicto +768 2459 768 1759 conicto +768 1061 1093 690 conicto +1418 320 2030 320 conicto +2340 320 2616 431 conicto +2892 543 3136 768 conicto +3136 256 lineto +2882 96 2599 16 conicto +2316 -64 2000 -64 conicto +1189 -64 722 424 conicto +256 913 256 1759 conicto +256 2607 722 3095 conicto +1189 3584 2000 3584 conicto +2320 3584 2603 3504 conicto +2887 3424 3136 3264 conicto +end_ol grestore +gsave 58.167080 34.140000 translate 0.035278 -0.035278 scale +start_ol +1473 2368 moveto +1117 2368 910 2094 conicto +704 1820 704 1344 conicto +704 868 909 594 conicto +1115 320 1473 320 conicto +1827 320 2033 595 conicto +2240 870 2240 1344 conicto +2240 1816 2033 2092 conicto +1827 2368 1473 2368 conicto +1472 2752 moveto +2040 2752 2364 2378 conicto +2688 2005 2688 1344 conicto +2688 685 2364 310 conicto +2040 -64 1472 -64 conicto +902 -64 579 310 conicto +256 685 256 1344 conicto +256 2005 579 2378 conicto +902 2752 1472 2752 conicto +end_ol grestore +gsave 58.559209 34.140000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 58.963823 34.140000 translate 0.035278 -0.035278 scale +start_ol +128 2688 moveto +586 2688 lineto +1408 432 lineto +2230 2688 lineto +2688 2688 lineto +1702 0 lineto +1114 0 lineto +128 2688 lineto +end_ol grestore +gsave 59.343467 34.140000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 59.738093 34.140000 translate 0.035278 -0.035278 scale +start_ol +1984 2304 moveto +1912 2337 1828 2352 conicto +1744 2368 1642 2368 conicto +1282 2368 1089 2127 conicto +896 1887 896 1437 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1030 2531 1245 2641 conicto +1460 2752 1767 2752 conicto +1811 2752 1864 2752 conicto +1917 2752 1982 2752 conicto +1984 2304 lineto +end_ol grestore +gsave 60.000345 34.140000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 56.807500 34.940000 translate 0.035278 -0.035278 scale +start_ol +1897 1936 moveto +2269 1862 2478 1627 conicto +2688 1392 2688 1046 conicto +2688 516 2331 226 conicto +1975 -64 1319 -64 conicto +1099 -64 865 -16 conicto +632 32 384 128 conicto +384 576 lineto +576 449 805 384 conicto +1035 320 1285 320 conicto +1720 320 1948 504 conicto +2176 688 2176 1039 conicto +2176 1363 1967 1545 conicto +1759 1728 1388 1728 conicto +1024 1728 lineto +1024 2112 lineto +1405 2112 lineto +1717 2112 1882 2250 conicto +2048 2388 2048 2648 conicto +2048 2915 1877 3057 conicto +1706 3200 1388 3200 conicto +1200 3200 985 3153 conicto +770 3107 512 3008 conicto +512 3456 lineto +773 3520 1001 3552 conicto +1229 3584 1431 3584 conicto +1952 3584 2256 3343 conicto +2560 3102 2560 2692 conicto +2560 2406 2387 2209 conicto +2215 2012 1897 1936 conicto +end_ol grestore +gsave 57.214619 34.940000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 57.706653 34.940000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 57.908960 34.940000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 58.400994 34.940000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 58.793124 34.940000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 59.042890 34.940000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 59.435019 34.940000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 1792 lineto +1552 1792 lineto +1881 1792 2060 1967 conicto +2240 2142 2240 2465 conicto +2240 2786 2060 2961 conicto +1881 3136 1552 3136 conicto +960 3136 lineto +448 3520 moveto +1552 3520 lineto +2145 3520 2448 3251 conicto +2752 2983 2752 2465 conicto +2752 1943 2448 1675 conicto +2145 1408 1552 1408 conicto +960 1408 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 59.792182 34.940000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 60.184311 34.940000 translate 0.035278 -0.035278 scale +start_ol +2688 1646 moveto +2688 0 lineto +2240 0 lineto +2240 1632 lineto +2240 2001 2093 2184 conicto +1947 2368 1654 2368 conicto +1302 2368 1099 2146 conicto +896 1925 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1053 2529 1266 2640 conicto +1479 2752 1757 2752 conicto +2217 2752 2452 2471 conicto +2688 2191 2688 1646 conicto +end_ol grestore +gsave 60.588925 34.940000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 60.983551 34.940000 translate 0.035278 -0.035278 scale +start_ol +448 3712 moveto +896 3712 lineto +896 0 lineto +448 0 lineto +448 3712 lineto +end_ol grestore +gsave 56.741250 35.740000 translate 0.035278 -0.035278 scale +start_ol +256 1536 moveto +1536 1536 lineto +1536 1152 lineto +256 1152 lineto +256 1536 lineto +end_ol grestore +gsave 56.971032 35.740000 translate 0.035278 -0.035278 scale +start_ol +512 2380 moveto +512 2816 lineto +3584 1703 lineto +3584 1305 lineto +512 192 lineto +512 628 lineto +2980 1502 lineto +512 2380 lineto +end_ol grestore +gsave 57.508029 35.740000 translate 0.035278 -0.035278 scale +start_ol +end_ol grestore +gsave 57.710336 35.740000 translate 0.035278 -0.035278 scale +start_ol +960 3136 moveto +960 384 lineto +1536 384 lineto +2266 384 2605 716 conicto +2944 1048 2944 1764 conicto +2944 2475 2605 2805 conicto +2266 3136 1536 3136 conicto +960 3136 lineto +448 3520 moveto +1463 3520 lineto +2493 3520 2974 3094 conicto +3456 2669 3456 1764 conicto +3456 853 2972 426 conicto +2488 0 1463 0 conicto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 58.202370 35.740000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 58.594499 35.740000 translate 0.035278 -0.035278 scale +start_ol +896 3456 moveto +896 2688 lineto +1792 2688 lineto +1792 2368 lineto +896 2368 lineto +896 902 lineto +896 572 984 478 conicto +1073 384 1345 384 conicto +1792 384 lineto +1792 0 lineto +1345 0 lineto +836 0 642 194 conicto +448 389 448 902 conicto +448 2368 lineto +128 2368 lineto +128 2688 lineto +448 2688 lineto +448 3456 lineto +896 3456 lineto +end_ol grestore +gsave 58.844266 35.740000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 59.236395 35.740000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +2496 3520 lineto +2496 3136 lineto +960 3136 lineto +960 2112 lineto +2368 2112 lineto +2368 1728 lineto +960 1728 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 59.556094 35.740000 translate 0.035278 -0.035278 scale +start_ol +1984 2304 moveto +1912 2337 1828 2352 conicto +1744 2368 1642 2368 conicto +1282 2368 1089 2127 conicto +896 1887 896 1437 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1030 2531 1245 2641 conicto +1460 2752 1767 2752 conicto +1811 2752 1864 2752 conicto +1917 2752 1982 2752 conicto +1984 2304 lineto +end_ol grestore +gsave 59.818346 35.740000 translate 0.035278 -0.035278 scale +start_ol +1622 1344 moveto +1104 1344 904 1225 conicto +704 1106 704 818 conicto +704 589 854 454 conicto +1004 320 1262 320 conicto +1618 320 1833 572 conicto +2048 825 2048 1244 conicto +2048 1344 lineto +1622 1344 lineto +2496 1513 moveto +2496 0 lineto +2048 0 lineto +2048 384 lineto +1900 154 1679 45 conicto +1458 -64 1138 -64 conicto +733 -64 494 162 conicto +256 389 256 769 conicto +256 1213 555 1438 conicto +854 1664 1448 1664 conicto +2048 1664 lineto +2048 1712 lineto +2048 2025 1851 2196 conicto +1655 2368 1300 2368 conicto +1074 2368 860 2320 conicto +646 2272 448 2176 conicto +448 2560 lineto +687 2656 912 2704 conicto +1138 2752 1351 2752 conicto +1927 2752 2211 2444 conicto +2496 2137 2496 1513 conicto +end_ol grestore +gsave 60.210475 35.740000 translate 0.035278 -0.035278 scale +start_ol +2549 2204 moveto +2714 2485 2942 2618 conicto +3171 2752 3482 2752 conicto +3899 2752 4125 2464 conicto +4352 2177 4352 1646 conicto +4352 0 lineto +3904 0 lineto +3904 1632 lineto +3904 2006 3769 2187 conicto +3634 2368 3356 2368 conicto +3017 2368 2820 2146 conicto +2624 1925 2624 1542 conicto +2624 0 lineto +2176 0 lineto +2176 1632 lineto +2176 2008 2041 2188 conicto +1906 2368 1624 2368 conicto +1289 2368 1092 2145 conicto +896 1922 896 1542 conicto +896 0 lineto +448 0 lineto +448 2688 lineto +896 2688 lineto +896 2304 lineto +1046 2534 1255 2643 conicto +1465 2752 1753 2752 conicto +2044 2752 2247 2611 conicto +2451 2471 2549 2204 conicto +end_ol grestore +gsave 60.832386 35.740000 translate 0.035278 -0.035278 scale +start_ol +2752 1480 moveto +2752 1280 lineto +704 1280 lineto +733 811 978 565 conicto +1223 320 1660 320 conicto +1914 320 2152 384 conicto +2390 448 2624 576 conicto +2624 192 lineto +2388 67 2140 1 conicto +1893 -64 1639 -64 conicto +1001 -64 628 309 conicto +256 683 256 1320 conicto +256 1979 613 2365 conicto +970 2752 1576 2752 conicto +2120 2752 2436 2410 conicto +2752 2068 2752 1480 conicto +2304 1600 moveto +2299 1950 2099 2159 conicto +1900 2368 1572 2368 conicto +1200 2368 976 2166 conicto +753 1964 719 1597 conicto +2304 1600 lineto +end_ol grestore +gsave 58.075000 37.625000 translate 0.035278 -0.035278 scale +start_ol +1536 3712 moveto +1211 3163 1053 2626 conicto +896 2090 896 1538 conicto +896 987 1055 446 conicto +1214 -94 1536 -640 conicto +1148 -640 lineto +797 -79 622 462 conicto +448 1004 448 1538 conicto +448 2070 621 2609 conicto +794 3149 1148 3712 conicto +1536 3712 lineto +end_ol grestore +gsave 58.324767 37.625000 translate 0.035278 -0.035278 scale +start_ol +2140 1632 moveto +2295 1581 2442 1413 conicto +2590 1245 2738 952 conicto +3200 0 lineto +2709 0 lineto +2250 893 lineto +2077 1243 1915 1357 conicto +1753 1472 1473 1472 conicto +960 1472 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +1552 3520 lineto +2157 3520 2454 3265 conicto +2752 3010 2752 2496 conicto +2752 2161 2595 1939 conicto +2439 1718 2140 1632 conicto +960 3136 moveto +960 1856 lineto +1552 1856 lineto +1893 1856 2066 2019 conicto +2240 2182 2240 2498 conicto +2240 2815 2066 2975 conicto +1893 3136 1552 3136 conicto +960 3136 lineto +end_ol grestore +gsave 58.744371 37.625000 translate 0.035278 -0.035278 scale +start_ol +1696 3047 moveto +1042 1280 lineto +2352 1280 lineto +1696 3047 lineto +1424 3520 moveto +1970 3520 lineto +3328 0 lineto +2827 0 lineto +2502 896 lineto +897 896 lineto +572 0 lineto +64 0 lineto +1424 3520 lineto +end_ol grestore +gsave 59.181463 37.625000 translate 0.035278 -0.035278 scale +start_ol +448 3520 moveto +1202 3520 lineto +2079 1148 lineto +2961 3520 lineto +3712 3520 lineto +3712 0 lineto +3200 0 lineto +3200 3095 lineto +2314 704 lineto +1846 704 lineto +960 3095 lineto +960 0 lineto +448 0 lineto +448 3520 lineto +end_ol grestore +gsave 59.733441 37.625000 translate 0.035278 -0.035278 scale +start_ol +384 3712 moveto +772 3712 lineto +1123 3149 1297 2609 conicto +1472 2070 1472 1538 conicto +1472 1004 1297 462 conicto +1123 -79 772 -640 conicto +384 -640 lineto +706 -94 865 446 conicto +1024 987 1024 1538 conicto +1024 2090 865 2626 conicto +706 3163 384 3712 conicto +end_ol grestore +showpage diff --git a/docs/slides/png/workflow_data_conversion/workflow_data_conversion.jpg b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.jpg new file mode 100644 index 0000000..20d6bff Binary files /dev/null and b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.jpg differ diff --git a/docs/slides/png/workflow_data_conversion/workflow_data_conversion.pdf b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.pdf new file mode 100644 index 0000000..0349b43 Binary files /dev/null and b/docs/slides/png/workflow_data_conversion/workflow_data_conversion.pdf differ diff --git a/docs/slides/png/workflow_simulation.jpg b/docs/slides/png/workflow_simulation.jpg new file mode 100644 index 0000000..cba20d8 Binary files /dev/null and b/docs/slides/png/workflow_simulation.jpg differ diff --git a/docs/visualisation_scripts/code_examples/transformations/growth_rates.c b/docs/visualisation_scripts/code_examples/transformations/growth_rates.c new file mode 100644 index 0000000..9a62211 --- /dev/null +++ b/docs/visualisation_scripts/code_examples/transformations/growth_rates.c @@ -0,0 +1,166 @@ + + +/* + * \fn: sum_array + * \brief: Sum from start to end in reverse order, sum_array(&x, start, enc) = sum[t-start,...,t-end] + * \brief: QoQ sum: sum_array(&X,1,3)=sum[t-1,...,t-3] + * \brief: YoY sum: sum_array(&X,1,12)=sum[t-1,...,t-12] + */ +void sum_array(double_array * x, int start, int end, double * sum_out) +{ + int i,n; + double sum=0.0; + + n = (*x).size; + +/* + printf("\n\nSize in function: %d;", n); + for (i=0; i<(*x).size; i++) + printf(" %2.2f", (*x).array[i]); +*/ + if(0<=start && start<=end && end<=n-1) + { + //printf("\n\nCondition true: start %d end %d [0<=start (%d) && start<=end (%d) && end<=n-1 (%d)]", start, end, (0<=start), (start<=end), (end<=n-1)); + for(i=start; i<=end; i++) + { + sum += (*x).array[i]; + //printf("\nFor-loop sum %2.2f", sum); + } + + }else + { + fprintf(stderr, "\n\n WARNING: sum_array fails. Condition is false (1=true, 0=false): start %d end %d [0<=start (%d) && start<=end (%d) && end<=n-1 (%d)]\nExiting sum_array\n", start, end, (0<=start), (start<=end), (end<=n-1)); + } + + (*sum_out) = sum; +} + +/* + * \fn: sum_array_backwards + * \brief: Sum from start to end in reverse order, sum_array(&x, start, enc) = sum[t-start,...,t-end] + * \brief: QoQ sum: sum_array(&X,1,3)=sum[t-1,...,t-3] + * \brief: YoY sum: sum_array(&X,1,12)=sum[t-1,...,t-12] + */ +void sum_array_backwards(double_array * x, int start, int end, double * sum_out) +{ + int i,n; + double sum=0.0; + + n = (*x).size; + +/* + printf("\n\nSize in function: %d;", n); + for (i=0; i<(*x).size; i++) + printf(" %2.2f", (*x).array[i]); +*/ + if(0=n-end; i--) + { + sum += (*x).array[i]; + //printf("\nFor-loop sum %2.2f", sum); + } + }else + { + fprintf(stderr, "\n\nIT %d WARNING: sum_array_backwards fails. Condition is false (1=true, 0=false): start %d end %d [0<=start (%d) && start<=end (%d) && end<=n (%d)]\nExiting sum_array_backwards\n", DAY, start, end, (0``, ``<=``, ``>=``, ``==``. + +Example:: + + var1: [variable name, 'operator[value]'] + e.g. + var1: [total_credit, '>[700]'] # selects those values of total credit variable that are greater than 700 + + + var2: [variable name, 'operator1[value]','operator2[value]'] + e.g. + var2: [equity, '>[700]', '<[1500]'] # selects those values of total credit variable that are between 700 and 1500 + + +Usage Example:: + + plot1: + timeseries: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit, '>[700]'] + var2: [equity, '>[700]', '<[800]'] + + + + +Similarly, for desired **sets**, **runs**, **major** and **minor** axes, filter conditions can be specified as necessary. + +``set`` : Specify the set numbers to be processed. Input can be a list ( *Case I* ), or ( *esp. for long lists*) a custom way +to specify the range of values ( *Case II* ). + +Example:: + + Case I: + set: [val(1),val(2),...,val(N)] + e.g. + set: [1,2] # list with values 1 and 2 + + Case II: + set: [range,[val(1),val(N),step-size]] + e.g. + set: [range, [1,10,2]] # list with values from 1 to 10 with a step-size of 2 + + +``run`` : Specify the runs to process. Syntax similar to ``set`` above. + +``major`` : Specify the values from the major axis (time periods) to process. Syntax similar to ``set`` above. + +``minor`` : Specify the values from the minor axis (agent instances) to process. Syntax similar to ``set`` above. + +*Note:* The ``set``, ``run``, ``major``, and ``minor`` values are nested under "Plot-type" + +Example:: + + plot1: + timeseries: + agent: Bank + analysis: multiple_set + variables: + var1: [total_credit] + var2: [equity] + set: [1] + run: [1,2] + major: [range,[6020,26000,20]] + minor: [1,5,7] + +``summary``: Specify the type of statistical summary. This is also nested under Plot-type. + +**Possible values** + +* ``mean``: returns the mean + +* ``median``: returns the median + +* ``custom_quantile``: this allows to select an upper and a lower quantile, see below. + +* ``third_quartile``: returns the 75th percentile of the data set + +* ``first_quartile``: returns the 25th percentile of the data set + +* ``maximum``: returns maximum value of the data set + +* ``minimum``: returns minimum value of the data set + +* ``full``: returns the full data set as an ensemble distribution. + + +.. * ``no``, ``none``: any non-existent keyword can be used if no summary of the data set is needed; these will be ignored by the code. + +Example:: + + plot1: + timeseries: + summary: mean + +For the value ``custom_quantile`` the lower and upper quantiles should be specified as floats in [0,1]. + +Example:: + + plot2: + timeseries: + summary: custom_quantile + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 + +A typical main configuration file may look as follows:: + + i/o: + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # set up your input_path for the resulting plots, it's relative unless it starts with a '/' + input_path: 'data/visualisation' + + # set up your output_path for the resulting plots, it's relative unless it starts with a '/' + output_path: 'results' + + input_files: + CentralBank: CentralBank.h5 # please name the key as the agent name + Eurostat: Eurostat.h5 + Firm: Firm.h5 + + plot1: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [13] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + + plot2: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 + +plot_config.yaml +~~~~~~~~~ + +The *plot_config.yaml* file contains all the necessary configuration options for a plot. Whenever a plot is specified on the *config.yaml* file, the *plot_config.yaml* file is read for the necessary specifications of the plot. As such, some of the parameters from the *plot_config.yaml* file is explained below. +All options can also be found on the matplotlib website +https://matplotlib.org/api/_as_gen/matplotlib.pyplot.plot.html + +**Plot-key( i.e.** ``plot1`` **):** This string should be the same as the Plot-key in the *config.yaml* file, to make sure +the correct parameters are mapped to the respective plot. + +``number_plots``: Specifies how many plots will be output per variable for a particular agent type. +*Possible values:* ``one``, ``many``. + +``plot_format``: Specify filetype for the plot [png|pdf]. + +``plot_name``: Specify a filename manually. If you leave it out the filename will be autogenerated (plot_name='Agent_Variable') + +*Note:* In case of multiple plots, a numerical suffix (*in increasing order*) is added after the specified file name. + +``plot_legend``: Specify legend for the plot. + +``legend_loc``: Specify location of the legend, either inside the box or outside of it. *Possible values:* ``in``, ``out``. + +``legend_label``: Specify name for the lines in the plot. Can be any string value. + +``xaxis_label``: Specify label for the x-axis. Can be any combination of string values. + +``yaxis_label``: Specify label for the y-axis. Can be any combination of string values. + +``linestyle``: Specify line characteristic. *Possible values:* ``solid``, ``dashed``, ``dashdot``, ``dotted`` etc. + +``greyscale``: Specify to plot in greyscale. *Possible values:* ``True``, ``False``. + +A typical *plot_config.yaml* file might look like this:: + + plot1: + number_plots: one + plot_name: p1_first_plot + plot_format: png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + greyscale: True + + plot2: + number_plots: one + plot_format: pdf + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: red + +If an option is not specified, then the default settings are:: + + plot_legend = 'no' + legend_label = None + legend_location = 'best' + plot_type = None + number_plots = 'one' + plot_name = None + plot_format = 'png' + l_lim = None + u_lim = None + linestyle = 'solid' + marker = 4 + markerfacecolor = None + markersize = None + facecolors = None + plot_title = None + xaxis_label = None + yaxis_label = None + number_bins = 50 + histtype = 'bar' + stacked = False + normed = 1 + fill_between = False + fillcolor = 'black' + greyscale = False + number_bars = 5 + + + + +config_transform.yaml +~~~~~~~~~~~~~~ + +The *config_transform.yaml* file contains all the necessary configurations for any transformation specified on the *config.yaml* file. Whenever a transformation is specified on the *config.yaml* file, the *config_transform.yaml* file is read for the necessary +specifications of the plot. + +As such, some of the parameters from the *config_transform.yaml* file is explained below: + +**Plot-key( i.e.** ``plot1`` **):** This string should be the same as the Plot-key in the config.yaml file, to make sure +the correct parameters are mapped to the respective plot. + +*Note:* Although it is called Plot-key, the transform case is an exception and no plots are produced in transform case. + +``variables``: Variables from the particular agent-type that is to be transformed. The sub-hierarchy ``var1``, ``var2`` etc. allows +the input of multiple variables for any agent type. + +``transform_function``: The transformation function to apply for the given variables to produce the necessary transforms. + +*Possible functions:* + +- Quarterly growth rate (quarter on quarter, at quarterly frequency) ``q_o_q_q`` +- Quarterly growth rate (quarter on quarter, at annual frequency) ``q_o_q_a`` +- Monthly growth rate (month on month, at monthly frequency) ``m_o_m_m`` +- Monthly growth rate (month on month, at annual frequency) ``m_o_m_a`` +- Annual growth (year on year, at annual frequency) ``y_o_y_a`` +- Other custom functions + +*Note:* Other elementary functions such as **sum**, **difference**, **product**, and **division** can also be performed, which will be added as custom functions in a future release. + +``aggregate``: If the transformation is to be performed after calculating the summary stats, then a necessary aggregation method can be specified. + +*Possible values:* ``mean``, ``median``, ``maximum``, ``minimum``, ``custom_quantile``, ``upper_quartile``, ``lower_quartile``. + +``write_file``: Specify whether to write the transformation as a file. *Possible values:* ``yes``, ``no``. + +``output_path``: If the ``write_file`` option above is set to ``yes``, then a output path for the file needs to be specified. +Can be any valid filepath, as a string, including upto the filename. + +``hdf_groupname``: Specify the rootname for the HDF5 group name (internal hierarchy) for the transformed variable. Can be any valid string. + +A particular *config_transform.yaml* file may, therefore, look as follows:: + + plot2: + variables: + var1: total_credit + var2: equity + transform_function: q_o_q_q + aggregate: mean + new_variables: + var1: total_credit_q_o_q_q + var2: equity_q_o_q_q + write_file: yes + output_file_name: 'transformed.h5' + hdf_groupname: 'quarterly_growth_rates' + + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**References:** + + Matplotlib: https://matplotlib.org/ + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/visualisation_scripts/images/P1_ts_technology.png b/docs/visualisation_scripts/images/P1_ts_technology.png new file mode 100644 index 0000000..fd3545e Binary files /dev/null and b/docs/visualisation_scripts/images/P1_ts_technology.png differ diff --git a/docs/visualisation_scripts/images/P2_ts_total_debt.png b/docs/visualisation_scripts/images/P2_ts_total_debt.png new file mode 100644 index 0000000..f5b7863 Binary files /dev/null and b/docs/visualisation_scripts/images/P2_ts_total_debt.png differ diff --git a/docs/visualisation_scripts/images/P3_ts_output.png b/docs/visualisation_scripts/images/P3_ts_output.png new file mode 100644 index 0000000..85a0e16 Binary files /dev/null and b/docs/visualisation_scripts/images/P3_ts_output.png differ diff --git a/docs/visualisation_scripts/images/P4_ts_price.png b/docs/visualisation_scripts/images/P4_ts_price.png new file mode 100644 index 0000000..bba7577 Binary files /dev/null and b/docs/visualisation_scripts/images/P4_ts_price.png differ diff --git a/docs/visualisation_scripts/images/P5_ts_unemployment_rate.png b/docs/visualisation_scripts/images/P5_ts_unemployment_rate.png new file mode 100644 index 0000000..d029c2a Binary files /dev/null and b/docs/visualisation_scripts/images/P5_ts_unemployment_rate.png differ diff --git a/docs/visualisation_scripts/images/P6_ts_monthly_output.png b/docs/visualisation_scripts/images/P6_ts_monthly_output.png new file mode 100644 index 0000000..3cd6001 Binary files /dev/null and b/docs/visualisation_scripts/images/P6_ts_monthly_output.png differ diff --git a/docs/visualisation_scripts/images/P7_ts_price.png b/docs/visualisation_scripts/images/P7_ts_price.png new file mode 100644 index 0000000..e592e4e Binary files /dev/null and b/docs/visualisation_scripts/images/P7_ts_price.png differ diff --git a/TODO/doc/plots/fourpointfive.png b/docs/visualisation_scripts/plots/fourpointfive.png similarity index 100% rename from TODO/doc/plots/fourpointfive.png rename to docs/visualisation_scripts/plots/fourpointfive.png diff --git a/TODO/doc/plots/fourpointfour.png b/docs/visualisation_scripts/plots/fourpointfour.png similarity index 100% rename from TODO/doc/plots/fourpointfour.png rename to docs/visualisation_scripts/plots/fourpointfour.png diff --git a/TODO/doc/plots/fourpointthree.png b/docs/visualisation_scripts/plots/fourpointthree.png similarity index 100% rename from TODO/doc/plots/fourpointthree.png rename to docs/visualisation_scripts/plots/fourpointthree.png diff --git a/TODO/doc/plots/one_set_agent_sp_price_delay_0.png b/docs/visualisation_scripts/plots/one_set_agent_sp_price_delay_0.png similarity index 100% rename from TODO/doc/plots/one_set_agent_sp_price_delay_0.png rename to docs/visualisation_scripts/plots/one_set_agent_sp_price_delay_0.png diff --git a/docs/visualisation_scripts/plots/one_set_multiple_runs_hg_price_price.png b/docs/visualisation_scripts/plots/one_set_multiple_runs_hg_price_price.png new file mode 100644 index 0000000..042b192 Binary files /dev/null and b/docs/visualisation_scripts/plots/one_set_multiple_runs_hg_price_price.png differ diff --git a/TODO/doc/plots/one_set_multiple_runs_sp_price_delay_0.png b/docs/visualisation_scripts/plots/one_set_multiple_runs_sp_price_delay_0.png similarity index 100% rename from TODO/doc/plots/one_set_multiple_runs_sp_price_delay_0.png rename to docs/visualisation_scripts/plots/one_set_multiple_runs_sp_price_delay_0.png diff --git a/docs/visualisation_scripts/plots/one_set_multiple_runs_sp_price_output_0.png b/docs/visualisation_scripts/plots/one_set_multiple_runs_sp_price_output_0.png new file mode 100644 index 0000000..65dc5ec Binary files /dev/null and b/docs/visualisation_scripts/plots/one_set_multiple_runs_sp_price_output_0.png differ diff --git a/docs/visualisation_scripts/plots/one_set_multiple_runs_timeseries_price.png b/docs/visualisation_scripts/plots/one_set_multiple_runs_timeseries_price.png new file mode 100644 index 0000000..5564cb8 Binary files /dev/null and b/docs/visualisation_scripts/plots/one_set_multiple_runs_timeseries_price.png differ diff --git a/docs/visualisation_scripts/plots/one_set_multiple_runs_ts_quantile_0.png b/docs/visualisation_scripts/plots/one_set_multiple_runs_ts_quantile_0.png new file mode 100644 index 0000000..5284aca Binary files /dev/null and b/docs/visualisation_scripts/plots/one_set_multiple_runs_ts_quantile_0.png differ diff --git a/TODO/doc/plots/one_set_one_run_agentanalysis_timeseries_price.png b/docs/visualisation_scripts/plots/one_set_one_run_agentanalysis_timeseries_price.png similarity index 100% rename from TODO/doc/plots/one_set_one_run_agentanalysis_timeseries_price.png rename to docs/visualisation_scripts/plots/one_set_one_run_agentanalysis_timeseries_price.png diff --git a/docs/visualisation_scripts/plots/one_set_one_run_bp_price_price.png b/docs/visualisation_scripts/plots/one_set_one_run_bp_price_price.png new file mode 100644 index 0000000..0c61c34 Binary files /dev/null and b/docs/visualisation_scripts/plots/one_set_one_run_bp_price_price.png differ diff --git a/docs/visualisation_scripts/plots/one_set_one_run_hg_price_price.png b/docs/visualisation_scripts/plots/one_set_one_run_hg_price_price.png new file mode 100644 index 0000000..b3d5f41 Binary files /dev/null and b/docs/visualisation_scripts/plots/one_set_one_run_hg_price_price.png differ diff --git a/TODO/doc/plots/ts_multibatch_analysis_0.png b/docs/visualisation_scripts/plots/ts_multibatch_analysis_0.png similarity index 100% rename from TODO/doc/plots/ts_multibatch_analysis_0.png rename to docs/visualisation_scripts/plots/ts_multibatch_analysis_0.png diff --git a/TODO/doc/plots/two_set_multiple_runs_hg_price_price.png b/docs/visualisation_scripts/plots/two_set_multiple_runs_hg_price_price.png similarity index 100% rename from TODO/doc/plots/two_set_multiple_runs_hg_price_price.png rename to docs/visualisation_scripts/plots/two_set_multiple_runs_hg_price_price.png diff --git a/TODO/doc/two.rst b/docs/visualisation_scripts/readme.rst similarity index 75% rename from TODO/doc/two.rst rename to docs/visualisation_scripts/readme.rst index 18bca9c..bfd2dea 100644 --- a/TODO/doc/two.rst +++ b/docs/visualisation_scripts/readme.rst @@ -1,58 +1,52 @@ -.. _tutorial: +Folder structure +==== -Possible Uses -============= +There are several scripts, located in folder ``src/visualisation_scripts`` that deal with different steps during the data visualisation/transformation processes: -The module can be used to filter, transform, and visualize time series data, in multiple hierarchial levels. - -The different kinds of plots that are possible are: time series, box plot, scatter plot, histogram, and delay plot. - -Structure -========= - -There are several scripts, located in folder ``/src`` that deal with different steps during the data visualisation/transformation processes: - -- ``main.py`` : Contains code to read in the input data and primary parameter files, filter the data based on necessary conditions, and link the different Summary, Plot, and Transformation modules. +- ``main.py`` : Contains code to read in the input data and primary config files, filter the data based on necessary conditions, and link the different Summary, Plot, and Transformation modules. - ``summarystats.py`` : Takes in a Pandas dataframe, and computes the necessary summary as needed, and outputs the result as a Pandas dataframe. - ``plots.py``: Takes in a Pandas dataframe, and returns the necessary plots as specified. - ``transform.py`` : Takes in a Pandas dataframe, and returns/ writes to a file the necessary transformations. +- ``parameters.py`` : Contains code for main configuration and reads in config files. -The parameter files are contained in the folder named ``/config``. It contains three configuration files: +The config files are contained in the folder named ``src/visualisation_scripts/config``. It contains three configuration files: -- ``main.yaml`` : define i/o path, plot-types, agents and appropriate variables, filter options, desired summary -- ``plot.yaml`` : define plot properties i.e. name, legends, line style, fill etc. -- ``transform.yaml``: define variables to transform, type of data transformations, i/o info to store data in a file after transformations +- ``config.yaml`` : define i/o path, plot-types, agents and appropriate variables, filter options, desired summary +- ``plot_config.yaml`` : define plot properties i.e. name, legends, line style, fill etc. +- ``config_transform.yaml``: define variables to transform, type of data transformations, i/o info to store data in a file after transformations -Note: Further details on how to use the parameter (yaml) files can be found on the ``/parameters.rst`` file of the documentation. +Note: Further details on how to use the config (yaml) files can be found on the configuration file `config.rst` in this documentation. Testing ======= -To run the module, simply use: +To run the module, use: - $ python main.py parameter-path + $ python3 /src/visualisation_scripts/main.py -p config-path -where, parameter path is the path to the folder containing the yaml files +where config-path is the path to the folder containing the yaml files (in this git repository's folder structure it is ``FLAViz/src/visualisation_scripts/config/``) -Note: The module has only been tested under Unix/ Unix-like systems. It has not been tested for Windows and therefore +Note: The module has only been tested under Unix/ Unix-like systems. It has not been tested for Windows and therefore there is no guarentee of proper execution for such systems. Examples ======== -Once the necessary parameters are set, by following the instructions specified in *parameters.rst* file, the module can be run to get the desired results. To demonstrate some of the functionalities, +Once the necessary parameters are set, by following the instructions specified in the configuration file `config.rst`, the module can be run to get the desired results. To demonstrate some of the functionalities, the parameters of the configuration files are shown below, along with the plots they yield. -Using the dataset , and the following parameter settings, the following plots can be produced. +Using the test_data_, and the following config settings, the following plots can be produced. + +.. _test_data: https://github.com/svdhoog/FLAViz/tree/master/data/visualisation Timeseries ========== **Example 1.1** (*For agent Firm, one set, ten runs, eighty instances, plotted in a single plot* ): -main.yaml:: +config.yaml:: plot1: timeseries: @@ -63,14 +57,14 @@ main.yaml:: set: [13] run: [range,[1,10]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: mean -plot.yaml:: +plot_config.yaml:: plot1: number_plots: one - plot_name: one_set_multiple_runs_timeseries.png + plot_format: png plot_legend: yes legend_location: best xaxis_label: Time @@ -89,7 +83,7 @@ plot.yaml:: **Example 1.2** (*For agent Firm, one set, one run, quantile values of of distribution over eighty instances, plotted in a single plot* ): -main.yaml:: +config.yaml:: plot2: timeseries: @@ -100,24 +94,24 @@ main.yaml:: set: [10] run: [1] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: custom_quantile - quantile_values: - lower_quantile : 0.20 - upper_quantile : 0.80 + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 -plot.yaml:: +plot_config.yaml:: plot2: number_plots: one - plot_name: one_set_multiple_runs_ts_quantile.png + plot_format: png plot_legend: yes legend_location: best xaxis_label: Time yaxis_label: price linestyle: solid marker: None - fill_between_quartiles: yes + fill_between: yes fillcolor: red @@ -130,7 +124,7 @@ plot.yaml:: **Example 1.3** (*For agent Firm, four sets, twenty runs each, eighty instances, quantile values plotted in a single plot* ): -main.yaml:: +config.yaml:: plot3: timeseries: @@ -141,25 +135,24 @@ main.yaml:: set: [10,13,16,17] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: custom_quantile - quantile_values: - lower_quantile : 0.20 - upper_quantile : 0.80 + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 -plot.yaml:: +plot_config.yaml:: plot3: number_plots: one - plot_name: ts_multibatch_analysis.png plot_legend: yes legend_location: best x-axis label: Time y-axis label: price linestyle: solid marker: None - fill_between_quartiles: yes + fill_between: yes fillcolor: darkgreen @@ -173,7 +166,7 @@ plot.yaml:: **Example 1.4** (*For agent Firm, one set, one run, twenty instances, timeseries plot of one variable* ): -main.yaml:: +config.yaml:: plot4: timeseries: @@ -184,16 +177,15 @@ main.yaml:: set: [13] run: [1] major: [range,[6020,12500,20]] - minor: [range,[1,20]] + minor: [range,[1,20]] summary: mean - -**Note:** In case where analysis: agent, the full set is plotted, so it is not necessary to specify summary. The distribution over agent-instances can be computed by calling multiple batch analysis, with a single set value and a single run value. -plot.yaml:: +**Note:** In case where analysis: agent, the full set is plotted, so it is not necessary to specify summary. The distribution over agent-instances can be computed by calling multiple batch analysis, with a single set value and a single run value. + +plot_config.yaml:: plot4: number_plots: one - plot_name: one_set_one_run_agentanalysis_timeseries.png plot_legend: no legend_location: best xaxis_label: Time @@ -216,7 +208,7 @@ Boxplot **Example 2.1** (*For agent Firm, one set, one run, eighty instances, boxplot* ): -main.yaml:: +config.yaml:: plot5: boxplot: @@ -230,11 +222,10 @@ main.yaml:: minor: [range,[1,80]] summary: full -plot.yaml:: +plot_config.yaml:: plot5: number_plots: one - plot_name: one_set_one_run_bp_price.png plot_legend: yes legend_label: (Agent = Firm, var = Price) legend_location: best @@ -249,14 +240,14 @@ plot.yaml:: :scale: 50 % :alt: alternate text :align: right - - + + Scatterplot =========== **Example 3.1** (*For agent Firm, one set, twenty runs, averages of eighty instances, scatterplot of the ensemble of two variables* ): -main.yaml:: +config.yaml:: plot6: scatterplot: @@ -265,18 +256,17 @@ main.yaml:: variables: var1: [price] var2: [output] - delay: no + delay: no set: [13] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: mean -plot.yaml:: +plot_config.yaml:: plot6: number_plots: one - plot_name: one_set_multiple_runs_sp_price_output.png plot_legend: yes legend_location: best legend_label: price vs. output @@ -294,7 +284,7 @@ plot.yaml:: **Example 3.2** (*For agent Firm, one set, twenty runs, eighty instances, scatterplot for the agent level of one variable with delay* ): -main.yaml:: +config.yaml:: plot7: scatterplot: @@ -302,18 +292,17 @@ main.yaml:: analysis: agent variables: var1: [price] - delay: yes + delay: yes set: [13] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: full -plot.yaml:: +plot_config.yaml:: plot7: number_plots: one - plot_name: one_set_agent_sp_price_delay.png plot_legend: no legend_location: best legend_label: price delay vs. price @@ -333,7 +322,7 @@ plot.yaml:: **Example 3.3** (*For agent Firm, one set, twenty runs each, eighty instances each, delay plot for one variable* ): -main.yaml:: +config.yaml:: plot8: scatterplot: @@ -345,14 +334,13 @@ main.yaml:: set: [13] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: mean -plot.yaml:: +plot_config.yaml:: plot8: number_plots: one - plot_name: one_set_multiple_runs_sp_price_delay.png plot_legend: yes legend_location: best legend_label: price delay vs. price @@ -373,7 +361,7 @@ Histogram **Example 4.1** (*For agent Firm, one set, one run, eighty instances, histogram of population distribution of one variable* ): -main.yaml:: +config.yaml:: plot9: histogram: @@ -384,19 +372,18 @@ main.yaml:: set: [10] run: [1] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: mean -plot.yaml:: +plot_config.yaml:: plot9: number_plots: one - plot_name: one_set_one_run_hg_price.png plot_title: (Agent = Firm, var = Price) number_bins: 50 histtype: bar plot_legend: yes - fill: yes + fill_between: yes stacked: False legend_location: best xaxis_label: xlabel @@ -413,7 +400,7 @@ plot.yaml:: **Example 4.2** (*For agent Firm, one set, twenty runs, eighty instances, histogram of population distribution of one variable* ): -main.yaml:: +config.yaml:: plot10: histogram: @@ -424,18 +411,17 @@ main.yaml:: set: [10] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: mean # Note: the summary is mean value, over each level, gives hist of distribution of means -plot.yaml:: +plot_config.yaml:: plot10: number_plots: one - plot_name: one_set_multiple_runs_hg_price.png plot_title: (Agent = Firm, var = Price) number_bins: 50 histtype: bar plot_legend: yes - fill: yes + fill_between: yes stacked: False legend_location: best xaxis_label: xlabel @@ -452,7 +438,7 @@ plot.yaml:: **Example 4.3** (*For agent Firm, histogram of ensemble distribution of one variable across twenty runs* ): -main.yaml:: +config.yaml:: plot11: histogram: @@ -463,19 +449,18 @@ main.yaml:: set: [10] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: full # Note: here the histogram is over the entire distribution -plot.yaml:: +plot_config.yaml:: plot11: number_plots: one - plot_name: one_set_multiple_runs_hg_price.png plot_title: (Agent = Firm, var = Price) number_bins: 50 histtype: bar plot_legend: yes - fill: yes + fill_between: yes stacked: False legend_location: best xaxis_label: xlabel @@ -495,7 +480,7 @@ plot.yaml:: **Example 4.4** (*For agent Firm, two sets, twenty runs, eighty instances, histogram of population distribution of one variable* ): -main.yaml:: +config.yaml:: plot12: histogram: @@ -506,19 +491,18 @@ main.yaml:: set: [10, 13] run: [range,[1,20]] major: [range,[6020,12500,20]] - minor: [range,[1,80]] + minor: [range,[1,80]] summary: mean -plot.yaml:: +plot_config.yaml:: plot12: number_plots: one - plot_name: two_sets_multiple_runs_hg_price.png plot_title: (Agent = Firm, var = Price) number_bins: 50 histtype: step plot_legend: yes - fill: no + fill_between: no stacked: False legend_location: best xaxis_label: xlabel @@ -531,7 +515,7 @@ plot.yaml:: :scale: 50 % :alt: alternate text :align: right - + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ **References:** @@ -543,7 +527,3 @@ plot.yaml:: HDF5: https://support.hdfgroup.org/HDF5/ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - - diff --git a/results/boxplot/plot5_Firm_price.png b/results/boxplot/plot5_Firm_price.png new file mode 100644 index 0000000..6d71d68 Binary files /dev/null and b/results/boxplot/plot5_Firm_price.png differ diff --git a/results/histogram/plot10_Firm_price.png b/results/histogram/plot10_Firm_price.png new file mode 100644 index 0000000..a4638c4 Binary files /dev/null and b/results/histogram/plot10_Firm_price.png differ diff --git a/results/histogram/plot11_Firm_price.png b/results/histogram/plot11_Firm_price.png new file mode 100644 index 0000000..3cfcbc5 Binary files /dev/null and b/results/histogram/plot11_Firm_price.png differ diff --git a/results/histogram/plot12_Firm_price.png b/results/histogram/plot12_Firm_price.png new file mode 100644 index 0000000..581bd86 Binary files /dev/null and b/results/histogram/plot12_Firm_price.png differ diff --git a/results/histogram/plot9_Firm_price.png b/results/histogram/plot9_Firm_price.png new file mode 100644 index 0000000..05453e9 Binary files /dev/null and b/results/histogram/plot9_Firm_price.png differ diff --git a/results/histogram/plot9_Firm_technology.png b/results/histogram/plot9_Firm_technology.png new file mode 100644 index 0000000..6659ced Binary files /dev/null and b/results/histogram/plot9_Firm_technology.png differ diff --git a/results/scatterplot/plot6_Firm_output_price.png b/results/scatterplot/plot6_Firm_output_price.png new file mode 100644 index 0000000..5ef263e Binary files /dev/null and b/results/scatterplot/plot6_Firm_output_price.png differ diff --git a/results/scatterplot/plot7_Firm_output_delay.png b/results/scatterplot/plot7_Firm_output_delay.png new file mode 100644 index 0000000..e93942e Binary files /dev/null and b/results/scatterplot/plot7_Firm_output_delay.png differ diff --git a/results/scatterplot/plot7_Firm_price_delay.png b/results/scatterplot/plot7_Firm_price_delay.png new file mode 100644 index 0000000..89af77b Binary files /dev/null and b/results/scatterplot/plot7_Firm_price_delay.png differ diff --git a/results/scatterplot/plot8_Firm_price.png b/results/scatterplot/plot8_Firm_price.png new file mode 100644 index 0000000..708134b Binary files /dev/null and b/results/scatterplot/plot8_Firm_price.png differ diff --git a/results/timeseries/plot1_Firm_price.png b/results/timeseries/plot1_Firm_price.png new file mode 100644 index 0000000..9d19e28 Binary files /dev/null and b/results/timeseries/plot1_Firm_price.png differ diff --git a/results/timeseries/plot2_Firm_price.png b/results/timeseries/plot2_Firm_price.png new file mode 100644 index 0000000..7337a42 Binary files /dev/null and b/results/timeseries/plot2_Firm_price.png differ diff --git a/results/timeseries/plot2_Firm_total_debt.png b/results/timeseries/plot2_Firm_total_debt.png new file mode 100644 index 0000000..5efffec Binary files /dev/null and b/results/timeseries/plot2_Firm_total_debt.png differ diff --git a/results/timeseries/plot3_Firm_price.png b/results/timeseries/plot3_Firm_price.png new file mode 100644 index 0000000..45d0f4a Binary files /dev/null and b/results/timeseries/plot3_Firm_price.png differ diff --git a/results/timeseries/plot3_Firm_technology.png b/results/timeseries/plot3_Firm_technology.png new file mode 100644 index 0000000..4842f81 Binary files /dev/null and b/results/timeseries/plot3_Firm_technology.png differ diff --git a/results/timeseries/plot3_Firm_total_debt.png b/results/timeseries/plot3_Firm_total_debt.png new file mode 100644 index 0000000..c1b34a4 Binary files /dev/null and b/results/timeseries/plot3_Firm_total_debt.png differ diff --git a/results/timeseries/plot4_Firm_price.png b/results/timeseries/plot4_Firm_price.png new file mode 100644 index 0000000..d980d8e Binary files /dev/null and b/results/timeseries/plot4_Firm_price.png differ diff --git a/src/data_conversion_old/README.md b/src/data_conversion_old/README.md deleted file mode 100644 index c5f069f..0000000 --- a/src/data_conversion_old/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Note on usage of the python scripts - -1. DB_HDF5 - - - These scripts convert SQLite database files to HDF5 files. - - There are two versions of the script currently, each having a slightly different functionalty. - a. The script "db_hdf5_v1.py" creates HDF5 files from corresponding SQLite db files in many-to-one fashion. So, for all db files present in one input folder, one equivalent HDF5 file is created in the output folder. For input folders containing more hierarchies, one HDF5 file is created for contents of each subfolder. - b. The script "db_hdf5_v2.py"ncreates HDF5 files from corresponding SQLite db files in one-to-one fashion. So, for each db file present in the input folder, equivalent HDF5 file is created in the output folder. To combine db files into a single HDF5 file , use version 1 of script named db_hdf5_v1.py instead. - - The script is refactored for Python3. Any help regarding the execution of the script can be obtained by using the following command in any terminal window: - - python3 db_hdf5_v1.py -h - - python3 db_hdf5_v2.py -h - - There is an option to pass-in a desired output folder for the output. - - Alternatively, by default the output is either created in the folder containing the script, or in the same folder that has the input folder. You can adjust the default option for output by commenting out the appropriate line in the script. Please look for the section "Set output parameters" in the script and comment out the option that you do not prefer. Currently, it is set such that the output folder is created in the folder containing the input folder by default. - -#### Note: There is no difference in the size of combined HDF5 (many-to-one generated by db_hdf5_v1.py) and separate HDF5 (one-to-one generated by db_hdf5_v2.py) files, even when compressing. However, separate HDF5 files are easier to manipulate when restructuring the HDF5 into files per Agent-type (i.e. Bank.h5, Eurostat.h5 and so on). Therefore, use of db_hdf5_v2.py is recommended for simplicity. - -2. XML_HDF5 - - - This script converts XML files to HDF5 files. - - This script creates HDF5 files from corresponding XML files in many-to-one fashion. So, for all xml files present in one input folder, one equivalent HDF5 file is created in the output folder. For input folders containing more hierarchies, one HDF5 file is created for contents of each subfolder. - - The script runs on Python2. Any help regarding the execution of the script can be obtained by using the following command in any terminal window: - - python xml_hdf5.py -h - - There is an option to pass-in a desired output folder for the output. - - Alternatively, by default the output is either created in the folder containing the script, or in the same folder that has the input folder. You can adjust the default option for output by commenting out the appropriate line in the script. Please look for the section "Set output parameters" in the script and comment out the option that you do not prefer. Currently, it is set such that the output folder is created in the folder containing the input folder by default. - -3. DB_XML - - - This script converts SQlite Db files to XML files. - - This script creates XML files from corresponding DB files in one-to-many fashion. For each DB file, a folder is created and multiple corresponding XML files are created inside the folder, based on time samples. - - The script runs on Python2. Any help regarding the execution of the script can be obtained by using the following command in any terminal window: - - python genxml.py -h - - There is an option to pass-in a desired output folder for the output. - - Alternatively, by default the output is either created in the folder containing the script, or in the same folder that has the input folder. You can adjust the default option for output by commenting out the appropriate line in the script. Please look for the section "Set output parameters" in the script and comment out the option that you do not prefer. Currently, it is set such that the output folder is created in the folder containing the input folder by default. diff --git a/src/data_conversion_old/merge_agentwise.py b/src/data_conversion_old/merge_agentwise.py deleted file mode 100755 index 6f4cfe3..0000000 --- a/src/data_conversion_old/merge_agentwise.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 - -import sqlite3, sys, glob, os, argparse, errno -import pandas as pd - -pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on - -DB_SUFFIX = '.h5' -agentname = "Eurostat" #TODO: make this a list and read all agents from list - -def write_hdf(fname, input_dbfolder): - set_run_name = fname[len(os.path.normpath(input_dbfolder))+3:-3] - print (set_run_name) - # Read the input HDF5 file in memory - store = pd.io.pytables.HDFStore(fname) - # Filtering out each agent types from the input HDF5 file - for key in store.keys(): - if agentname in key: - frame = store.select(key) - store_out[set_run_name] = frame - store.close() - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - reply = raw_input("Specified output directory already exists!! Delete existing directory named <<"+os.path.basename(d)+">> and all its contents? [y/n] ") - if reply in ['y', 'Y', 'yes']: - try: - os.system('rm -r '+ d) - "Directory named <<"+os.path.basename(d)+ ">> and all its contents deleted!!" - # Make new output folder - try: - os.makedirs(d) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - except: - error("- Could not delete directory <<" +os.path.basename(d)+">>. Directory may contain additional files, remove files manually and try again!") - else: - replytwo = raw_input("Continue & write output files inside existing directory: <<"+os.path.basename(d)+">> ? WARNING: This will overwrite old files having same name, if present in the folder! [y/n]: ") - if not replytwo in ['y', 'Y', 'yes']: - try: - print ("Please remove or rename the existing directory <<"+os.path.basename(d)+">> and try again, or choose a different directory for the output") - sys.exit() - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - else: - os.makedirs(d) - - -# Function to print out the error messages,if any, and exit -def error(mesg): - print (">>>>> (Error): %s" % mesg) - sys.exit(1) - - - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='db_hdf5_v2.py', description='Converts the SQLite database files to HDF5 files. For each db files, creates an equivalent HDF5 file.') - parser.add_argument('dbpath', help='Path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - - args = parser.parse_args() - - # Set input parameters - input_dbfolder = '' - input_dbfolder = args.dbpath[0] - if input_dbfolder == "./": - cwd = os.getcwd() - print ("- Python script and input csv files both inside the same folder <<" +cwd+">>. Expected at least one level of hierarchy!") - sys.exit(1) - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): - dir_list.append(dirpath) - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - - # Set output parameters - targetFolder = '' - if args.outpath: - targetFolder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - targetFolder = './output_'+os.path.basename(os.path.normpath(input_dbfolder)) # For output folder in the same folder where Python script is located. - #targetFolder = os.path.abspath(os.path.join(os.path.normpath(input_dbfolder), os.pardir))+'/output_'+os.path.basename(os.path.normpath(input_dbfolder)) # For output folder in the same folder where input folder is located - - - # Function call to check if the output folder already exists, and create if not present - dir_check(targetFolder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - #print (arg, end=' ') - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print() - else: - statusprint = lambda *a: None - - # Process each folder in the input directory - processed_folders = 0 - statusprint('\n- Total number of folders: '+ str(F)+'\n') - for i in range(N,len(dir_list)): - n = len(os.path.normpath(input_dbfolder)) + 2 - statusprint('- Started processing folder: '+os.path.basename(dir_list[i])) - if N == 1: - output_folder = targetFolder + dir_list[i][n:] - dir_check(output_folder) - else: - output_folder = targetFolder - - # Populate the list with all sql file names in the folder - db_file_list = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): - db_file_list.append(fname) - statusprint('- Total number of files within folder: '+ str(len(db_file_list))) - # Generate panels - processed_files =[] - #print (db_file_list) - outFileName = output_folder + '/' + agentname +'.h5' - store_out = pd.HDFStore(outFileName, 'w', chunksize = 500, complevel = 1, complib ='bzip2', fletcher32 = True) - for fname in db_file_list: - verboseprint('\n- Started processing: '+os.path.basename(fname)) - write_hdf(fname, input_dbfolder) - processed_files.append(fname) - percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), - store_out.close() - statusprint('- Finished processing folder: '+os.path.basename(dir_list[i])+'\n') - processed_folders = processed_folders+1 - f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - diff --git a/src/data_conversion_old/sql_hdf5/TODO.txt b/src/data_conversion_old/sql_hdf5/TODO.txt deleted file mode 100644 index 8f8a049..0000000 --- a/src/data_conversion_old/sql_hdf5/TODO.txt +++ /dev/null @@ -1 +0,0 @@ -If output folder specified as current folder "./", asks to delete the folder, remove this. Just include a warning saying whether you want to overwrite if old files present. diff --git a/src/data_conversion_old/sql_hdf5/db_hdf5_v1.py b/src/data_conversion_old/sql_hdf5/db_hdf5_v1.py deleted file mode 100644 index ee25d02..0000000 --- a/src/data_conversion_old/sql_hdf5/db_hdf5_v1.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python3 - -################################################################################################################# -## This script creates HDF5 files from corresponding SQLite db files in many-to-one fashion. So, for all db files -## present in one input folder, one equivalent HDF5 file is created in the output folder. For input folders -## containing more hierarchies, one HDF5 file is created for contents of each subfolder. -################################################################################################################# - -import sqlite3, sys, glob, os, argparse, errno -import pandas as pd -from glob import glob as g -pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/ appended to' later on -DB_SUFFIX = '.db' - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - reply = input("Specified output directory already exists!! Delete existing directory named <<"+os.path.basename(d)+">> and all its contents? [y/n] ") - if reply in ['y', 'Y', 'yes']: - try: - os.system('rm -r '+ d) - print("Directory named <<"+os.path.basename(d)+ ">> and all its contents deleted!!") - # Make new output folder - try: - os.makedirs(d) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - except: - error("- Could not delete directory <<" +os.path.basename(d)+">>. Directory may contain additional files, remove files manually and try again!") - else: - replytwo = input("Continue & write output files inside existing directory: <<"+os.path.basename(d)+">> ? WARNING: This will overwrite old files having same name, if present in the folder! [y/n]: ") - if not replytwo in ['y', 'Y', 'yes']: - try: - print ("Please remove or rename the existing directory <<"+os.path.basename(d)+">> and try again, or choose a different directory for the output") - sys.exit() - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - else: - os.makedirs(d) - - -# Function to print out the error messages,if any, and exit -def error(mesg): - print(">>>>> (Error): %s" % mesg, file = sys.stderr) - sys.exit(1) - - -def agent_dataframe(agent_name): - agent_name = agent_name.decode('utf-8') - #df = pd.read_sql_query(("SELECT * from "+ agent_name), con) - df = pd.read_sql(("SELECT * from "+ agent_name), con) - - # Ignoring the empty tables - if df.empty == False: - #Converting the datatypes to make all of them uniform, to avoid pickling later on when writing to HDF - types = df.apply(lambda x: pd.lib.infer_dtype(x.values)) - - for col in types[types=='floating'].index: - df[col] = df[col].astype(str) - - for col in types[types=='mixed'].index: - df[col] = df[col].astype(str) - - for col in types[types=='unicode'].index: - df[col] = df[col].astype(str) - - # Keeping count of the individual agent instance - IterationList =[] - agtin = str('agent_inst').encode('utf-8') - IterationList = df['_ITERATION_NO'].tolist() - j=0 - agentinstance_index =[j] - for i in range(1,len(IterationList)): - if IterationList[i] == IterationList[i-1]: - j=j+1 - agentinstance_index.append(j) - else: - j=0 - agentinstance_index.append(j) - # Adding the agent instance count as a separate column to the dataframe - df[agtin] = agentinstance_index - # Creating hierarchial index for df with the iteration number, and agent instance count - df.set_index(['_ITERATION_NO',agtin], inplace=True) - return df - -def gendp(dframes): - dp = dframes.to_panel() - #pnl = dp.swapaxes(0,2, copy=True) #This will allow to swap the datapanel axes, if needed - return dp - -def gen_hdf(fname): - global con, cur, agent_name - con = None - try: - con = sqlite3.connect(fname) - cur = con.cursor() - - # Get the name of all the tables in the SQLite DB i.e. the agents - cur.execute("SELECT name from sqlite_master WHERE type='table';") - agent_list = [x[0].encode('ascii', 'ignore') for x in cur.fetchall()] - w = str('_iters_').encode('utf-8') - if w in agent_list: - agent_list.remove(w) - # Iterate through all the agent tables in the DB and add each to a unique Dataframe - for agent_name in agent_list: - verboseprint ('- Processing table values for: '+str(agent_name)) - dfa = agent_dataframe(agent_name) - verboseprint ('- Obtained Dataframe for: '+str(agent_name)) - # Checking and ignoring empty agent tables that might be in the DB - if dfa is not None: - verboseprint ('- Preparing HDF5 file for writing') - a = agent_name.decode('utf-8') - s = str(os.path.basename(fname))[:-3] +'/'+str(a) - # Storing the panels in the HDF Store - store[s] = gendp(dfa) - verboseprint ('- Successfully wrote: '+str(agent_name)+ ' into HDF5 file \n') - except sqlite3.Error as e: - print("Error ", e.args[0]) - sys.exit(1) - finally: - if con: - con.close() - del dfa - - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='db_hdf5_v1.py', description='Converts the SQLite database files to HDF5 files. Combines multiple db files into a single HDF5 file.') - parser.add_argument('dbpath', help='Path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - - args = parser.parse_args() - - # Set input parameters - input_dbfolder = '' - input_dbfolder = args.dbpath[0] - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): - dir_list.append(dirpath) - if len(dir_list) == 0: - error("- Make sure the specified input directory is an actual directory, and not a file!") - if os.getcwd() == os.path.abspath(input_dbfolder): - error("- BAD IDEA!!! Execution script and input db files both inside a single folder <<" +os.getcwd()+">>. Might cause a recursive loop leading to erroneous output in some cases. Expected at least one level of separation. Please keep the script somewhere else and retry!") - - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - - # Set output parameters - output_folder = '' - if args.outpath: - output_folder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - #output_folder = './output_'+os.path.basename(input_dbfolder) # Creates output folder in the same folder where Python script is located. - output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(input_dbfolder) # Creates output folder in the same folder where input folder is located - - # Function call to check if the output folder already exists, and create if not present - dir_check(output_folder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print(arg, end=' ') - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print() - else: - statusprint = lambda *a: None - processed_folders = 0 - statusprint('\n- Total number of folders: '+ str(F)+'\n') - for i in range(N,len(dir_list)): - statusprint('- Started processing folder: '+os.path.basename(dir_list[i])) - # Populate the list with all sql file names in the folder - db_file_list = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): - db_file_list.append(fname) - statusprint('- Total number of files in current folder: '+ str(len(db_file_list))) - - # Setting up HDF5 file for storing output - fname_tostore = output_folder +'/'+os.path.basename(dir_list[i])+'.h5' - verboseprint ('\n- Preparing HDF5 file for writing contents of input folder: '+os.path.basename(dir_list[i])) - store = pd.HDFStore(fname_tostore, 'w', complevel = 1, complib ='bzip2', fletcher32 = True) - - # Generate panels - processed_files =[] - for fname in db_file_list: - verboseprint ('\n- Started processing: '+os.path.basename(fname)) - - gen_hdf(fname) - processed_files.append(fname) - percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), - verboseprint ('- Successfully closed HDF5 file for: '+os.path.basename(dir_list[i])) - store.close() - statusprint('- Finished processing folder: '+ os.path.basename(dir_list[i])+'\n') - processed_folders = processed_folders+1 - f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - - - diff --git a/src/data_conversion_old/sql_hdf5/db_hdf5_v2.py b/src/data_conversion_old/sql_hdf5/db_hdf5_v2.py deleted file mode 100644 index d67354f..0000000 --- a/src/data_conversion_old/sql_hdf5/db_hdf5_v2.py +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env python3 - -################################################################################################################# -## This script creates HDF5 files from corresponding SQLite db files in one-to-one fashion. So, for each db file -## present in the input folder, equivalent HDF5 file is created in the output folder. To combine db files into -## a single HDF5 file , use version 1 of script named db_hdf5_v1.py instead -################################################################################################################# - -import sqlite3, sys, glob, os, argparse, errno -import pandas as pd -from glob import glob as g -pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on -DB_SUFFIX = '.db' - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - reply = input("Specified output directory already exists!! Delete existing directory named <<"+os.path.basename(d)+">> and all its contents? [y/n] ") - if reply in ['y', 'Y', 'yes']: - try: - os.system('rm -r '+ d) - print("Directory named <<"+os.path.basename(d)+ ">> and all its contents deleted!!") - # Make new output folder - try: - os.makedirs(d) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - except: - error("- Could not delete directory <<" +os.path.basename(d)+">>. Directory may contain additional files, remove files manually and try again!") - else: - replytwo = input("Continue & write output files inside existing directory: <<"+os.path.basename(d)+">> ? WARNING: This will overwrite old files having same name, if present in the folder! [y/n]: ") - if not replytwo in ['y', 'Y', 'yes']: - try: - print ("Please remove or rename the existing directory <<"+os.path.basename(d)+">> and try again, or choose a different directory for the output") - sys.exit() - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - else: - os.makedirs(d) - - -# Function to print out the error messages,if any, and exit -def error(mesg): - print(">>>>> (Error): %s" % mesg, file = sys.stderr) - sys.exit(1) - - -def agent_dataframe(agent_name): - agent_name = agent_name.decode('utf-8') - #df = pd.read_sql_query(("SELECT * from "+ agent_name), con) - df = pd.read_sql(("SELECT * from "+ agent_name), con) - - # Ignoring the empty tables - if df.empty == False: - #Converting the datatypes to make all of them uniform, to avoid pickling later on when writing to HDF - types = df.apply(lambda x: pd.lib.infer_dtype(x.values)) - - for col in types[types=='floating'].index: - df[col] = df[col].astype(str) - - for col in types[types=='mixed'].index: - df[col] = df[col].astype(str) - - for col in types[types=='unicode'].index: - df[col] = df[col].astype(str) - - # Keeping count of the individual agent instance - IterationList =[] - agtin = str('agent_inst').encode('utf-8') - IterationList = df['_ITERATION_NO'].tolist() - j=0 - agentinstance_index =[j] - for i in range(1,len(IterationList)): - if IterationList[i] == IterationList[i-1]: - j=j+1 - agentinstance_index.append(j) - else: - j=0 - agentinstance_index.append(j) - # Adding the agent instance count as a separate column to the dataframe - df[agtin] = agentinstance_index - # Creating hierarchial index for df with the iteration number, and agent instance count - df.set_index(['_ITERATION_NO',agtin], inplace=True) - return df - -def gendp(dframes): - dp = dframes.to_panel() - #pnl = dp.swapaxes(0,2, copy=True) # This will allow to swap the datapanel axes, if needed - return dp - -def gen_hdf(fname, output_folder): - outFileName = output_folder + '/' + os.path.basename(fname)[:-len(DB_SUFFIX)] - fname_tostore = outFileName +'.h5' - #verboseprint ('- Preparing HDF5 file: '+os.path.basename(fname_tostore)+'for writing') - store = pd.HDFStore(fname_tostore, 'w', chunksize = 500, complevel = 1, complib ='bzip2', fletcher32 = True) - global con, cur, agent_name - con = None - try: - con = sqlite3.connect(fname) - cur = con.cursor() - - # Get the name of all the tables in the SQLite DB i.e. the agents - cur.execute("SELECT name from sqlite_master WHERE type='table';") - agent_list = [x[0].encode('ascii', 'ignore') for x in cur.fetchall()] - w = str('_iters_').encode('utf-8') - if w in agent_list: - agent_list.remove(w) - # Iterate through all the agent tables in the DB and add each to a unique Dataframe - for agent_name in agent_list: - verboseprint ('- Processing table values for: '+str(agent_name)) - dfa = agent_dataframe(agent_name) - verboseprint ('- Obtained Dataframe for: '+str(agent_name)) - # Checking and ignoring empty agent tables that might be in the DB - if dfa is not None: - verboseprint ('- Preparing HDF5 file for writing') - a = agent_name.decode('utf-8') - # Storing the panels in the HDF Store - store[a] = gendp(dfa) - verboseprint ('- Successfully wrote: '+str(agent_name)+ ' into HDF5 file. \n') - except sqlite3.Error as e: - print("Error ", e.args[0]) - sys.exit(1) - finally: - if con: - con.close() - verboseprint ('- Successfully closed HDF5 file: '+os.path.basename(fname_tostore)+'\n') - store.close() - #verboseprint('- Done processing: ',os.path.basename(fname)) - del dfa - - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='db_hdf5_v2.py', description='Converts the SQLite database files to HDF5 files. For each db files, creates an equivalent HDF5 file.') - parser.add_argument('dbpath', help='Path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - - args = parser.parse_args() - - # Set input parameters - input_dbfolder = '' - input_dbfolder = args.dbpath[0] - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): - dir_list.append(dirpath) - if len(dir_list) == 0: - error("- Make sure the specified input directory is an actual directory, and not a file!") - if os.getcwd() == os.path.abspath(input_dbfolder): - error("- BAD IDEA!!! Execution script and input db files both inside a single folder <<" +os.getcwd()+">>. Might cause a recursive loop leading to erroneous output in some cases. Expected at least one level of separation. Please keep the script somewhere else and retry!") - - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - - # Set output parameters - output_folder = '' - if args.outpath: - output_folder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - #output_folder = './output_'+os.path.basename(input_dbfolder) # Creates output folder in the same folder where Python script is located. - output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(input_dbfolder) # Creates output folder in the same folder where input folder is located - - - # Function call to check if the output folder already exists, and create if not present - dir_check(output_folder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print(arg, end=' ') - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print() - else: - statusprint = lambda *a: None - - # Process each folder in the input directory - processed_folders = 0 - statusprint('\n- Total number of folders: '+ str(F)+'\n') - for i in range(N,len(dir_list)): - statusprint('- Started processing folder: '+os.path.basename(dir_list[i])) - # Populate the list with all sql file names in the folder - db_file_list = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): - db_file_list.append(fname) - statusprint('- Total number of files within folder: '+ str(len(db_file_list))) - # Generate panels - processed_files =[] - for fname in db_file_list: - verboseprint('\n- Started processing: '+os.path.basename(fname)) - - gen_hdf(fname, output_folder) - processed_files.append(fname) - percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), - statusprint('- Finished processing folder: '+os.path.basename(dir_list[i])+'\n') - processed_folders = processed_folders+1 - f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - - - - - - diff --git a/src/data_conversion_old/sql_xml/genxml.py b/src/data_conversion_old/sql_xml/genxml.py deleted file mode 100644 index 420f371..0000000 --- a/src/data_conversion_old/sql_xml/genxml.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python - -################################################################################################################# -## This script creates HDF5 files from corresponding SQLite db files in many-to-one fashion. So, for all db files -## present in one input folder, one equivalent HDF5 file is created in the output folder. For input folders -## containing more hierarchies, one HDF5 file is created for contents of each subfolder. -################################################################################################################# - -import sqlite3, sys, glob, os, argparse,errno - -DB_SUFFIX = '.db' -XML_ROOT = 'states' - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - reply = raw_input("Specified output directory already exists!! Delete existing directory named <<"+os.path.basename(d)+">> and all its contents? [y/n] ") - if reply in ['y', 'Y', 'yes']: - try: - os.system('rm -r '+ d) - print("Directory named <<"+os.path.basename(d)+ ">> and all its contents deleted!!") - # Make new output folder - try: - os.makedirs(d) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - except: - error("- Could not delete directory <<" +os.path.basename(d)+">>. Directory may contain additional files, remove files manually and try again!") - else: - replytwo = raw_input("Continue & write output files inside existing directory: <<"+os.path.basename(d)+">> ? WARNING: This will overwrite old files having same name, if present in the folder! [y/n]: ") - if not replytwo in ['y', 'Y', 'yes']: - try: - print ("Please remove or rename the existing directory <<"+os.path.basename(d)+">> and try again, or choose a different directory for the output") - sys.exit() - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - else: - os.makedirs(d) - -# Function to print out the error messages,if any, and exit -def error(mesg): - print >>sys.stderr, ">>>>> (Error): %s" % mesg - sys.exit(1) - -def convert2xml(fname, targetFolder): - # Create folder with sqlname as filename - verboseprint('\n-Found file: ',fname) - outFolderName = targetFolder + '/' + os.path.basename(fname)[:-len(DB_SUFFIX)] - dir_check(outFolderName) - verboseprint('-Successfully created output folder: ', outFolderName) - - con = None - try: - con = sqlite3.connect(fname) - cur = con.cursor() - - # Get its_names from the table _iters_ - verboseprint('-Extracting its_names: ',os.path.basename(fname)) - cur.execute("SELECT * from _iters_;") - its_list = [x[0] for x in cur.fetchall()] - verboseprint(its_list) - verboseprint('-Extract successful: its_name') - - # Create xml files - verboseprint('-Initializing XML files:') - for itsName in its_list: - xmlFile = open(outFolderName + '/' + str(itsName) + '.xml','w') - xmlFile.write('<' + XML_ROOT + '>' + '\n' + '' + str(itsName) +''+ '\n') - xmlFile.close() - verboseprint(' '+str(itsName)+'.xml'), - - # Get the name of all the tables - verboseprint('-Analysing database structure') - cur.execute("SELECT name from sqlite_master WHERE type='table';") - table_list = [x[0].encode('ascii', 'ignore') for x in cur.fetchall()] - if '_iters_' in table_list: table_list.remove('_iters_') - - for table_name in table_list: - agent_name = table_name - verboseprint('-Reading table: ',table_name) - # Get column names of the table - cur.execute("PRAGMA table_info("+ table_name +")") - - # Zeroth column contains '_ITERATION_NO' so we start from index 1 - variable_names = [x[1].encode('ascii', 'ignore') for x in cur.fetchall()] - iterNoColIdx = variable_names.index('_ITERATION_NO') - - xmlStr = "" - oldFileName = "" - xmlFile = None - for row in cur.execute("SELECT * from "+ table_name): - xmlStr = "" - fileName = outFolderName + '/' + str(row[iterNoColIdx]) + '.xml' - - # Check if file is already open, for reuse - if (fileName <> oldFileName): - if (xmlFile): - xmlFile.close() - xmlFile = open(fileName, 'a') - oldFileName = fileName - - i = 1 - for var in variable_names[1:]: - xmlStr = xmlStr + '<' + var + '>' + ('' if not row[i] else row[i]) + '' + '\n' - i = i + 1 - - xmlFile.write('\n'+ agent_name + '' + '\n' + xmlStr + '' + '\n') - verboseprint('-Writing into XML file: ',os.path.basename(fileName)) - if (xmlFile): - xmlFile.close() - - # Close the document root tag - for itsName in its_list: - xmlFile = open(outFolderName + '/' + str(itsName)+'.xml', 'a') - xmlFile.write('') - xmlFile.close() - - except sqlite3.Error, e: - print "Error ", e.args[0] - sys.exit(1) - finally: - if con: - con.close() - verboseprint('-Done processing: ',os.path.basename(fname), '\n') - - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='genxml.py', description='Converts the sqlite database to xml.') - parser.add_argument('dbpath', help='path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='get the total progress of the processing', action='store_true') - args = parser.parse_args() - - # Set input parameters - input_dbfolder = '' - input_dbfolder = args.dbpath[0] - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): - dir_list.append(dirpath) - if len(dir_list) == 0: - error("- Make sure the specified input directory is an actual directory, and not a file!") - if os.getcwd() == os.path.abspath(input_dbfolder): - error("- BAD IDEA!!! Execution script and input db files both inside a single folder <<" +os.getcwd()+">>. Might cause a recursive loop leading to erroneous output in some cases. Expected at least one level of separation. Please keep the script somewhere else and retry!") - - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - # Set output parameters - output_folder = '' - if args.outpath: - output_folder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - #output_folder = './output_'+os.path.basename(input_dbfolder) # Creates output folder in the same folder where Python script is located. - output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(input_dbfolder) # Creates output folder in the same folder where input folder is located - - - # Function call to check if the output folder already exists, and create if not present - dir_check(output_folder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print arg, - print - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print - else: - statusprint = lambda *a: None - - # Process each folder in the input directory - processed_folders = 0 - statusprint('\n- Total number of input folders: '+ str(F)+'\n') - for i in range(N,len(dir_list)): - statusprint('- Started processing folder: '+os.path.basename(dir_list[i])) - # Populate the list with all sql file names in the folder - sql_file_list = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): - sql_file_list.append(fname) - statusprint('Total number of files in current folder: '+ str(len(sql_file_list))) - - # Generate xml - processed_files =[] - for fname in sql_file_list: - verboseprint ('\n- Started processing: '+os.path.basename(fname)) - convert2xml(fname, output_folder) - processed_files.append(fname) - percent = round((float(len(processed_files))/len(sql_file_list))*100,2) - statusprint('Number processed files: '+str(len(processed_files))+', of total: '+str(len(sql_file_list))+' Progress:'+ str(percent) +'%'), - statusprint('- Finished processing folder: '+ os.path.basename(dir_list[i])+'\n') - processed_folders = processed_folders+1 - f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - - - - diff --git a/src/data_conversion_old/xml_hdf5/xml_hdf5.py b/src/data_conversion_old/xml_hdf5/xml_hdf5.py deleted file mode 100644 index de18d1a..0000000 --- a/src/data_conversion_old/xml_hdf5/xml_hdf5.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python - -################################################################################################################# -## This script creates HDF5 files from corresponding XML files in many-to-one fashion. So, for all xml files -## present in one input folder, one equivalent HDF5 file is created in the output folder. For input folders -## containing more hierarchies, one HDF5 file is created for contents of each subfolder. -################################################################################################################# - -import os, sys, re, argparse, glob, errno -import lxml.etree as ET -from xml.dom import minidom -from xml.parsers.expat import ExpatError -from glob import glob as g -import pandas as pd -pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/ appended to' later on -XML_SUFFIX = '.xml' - -def file_exist(fname): - f = g(fname) - if fname in f: return True - else: return False - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - reply = raw_input("Specified output directory already exists!! Delete existing directory named <<"+os.path.basename(d)+">> and all its contents? [y/n] ") - if reply in ['y', 'Y', 'yes']: - try: - os.system('rm -r '+ d) - print("Directory named <<"+os.path.basename(d)+ ">> and all its contents deleted!!") - # Make new output folder - try: - os.makedirs(d) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - except: - error("- Could not delete directory <<" +os.path.basename(d)+">>. Directory may contain additional files, remove files manually and try again!") - else: - replytwo = raw_input("Continue & write output files inside existing directory: <<"+os.path.basename(d)+">> ? WARNING: This will overwrite old files having same name, if present in the folder! [y/n]: ") - if not replytwo in ['y', 'Y', 'yes']: - try: - print ("Please remove or rename the existing directory <<"+os.path.basename(d)+">> and try again, or choose a different directory for the output") - sys.exit() - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - else: - os.makedirs(d) - -# Function to print out the error messages,if any, and exit -def error(mesg): - print >>sys.stderr, ">>>>> (Error): %s" % mesg - sys.exit(1) - -# Function to print out the error messages,if any -def warn(mesg): - print >>sys.stderr, ">> (Warning): %s" % mesg - -def get_node_text(node): - if len(node.childNodes) != 1 or node.childNodes[0].nodeType != node.TEXT_NODE: - error("getText() routine used on non-text XML node") - return node.childNodes[0].data.strip() - -# Function to generate a dataframe for the given agent from all the xml files in one folder -def gendf(files,agents): - global df, colmList - xmlTree = ET.parse(files) - root = xmlTree.getroot() - allxagent = root.findall('xagent') - count = 0 - xList = [] - colmList = [] - for xagent in allxagent: - # Counting the occurrance of a single agent type from xml file containing mixed agent types - count = count + 1 - for names in xagent: - if names.text in [agents]: - xList.append(count) - colmList = [] - # Getting the variable names for that particular agent - for subagent in xagent: - colmList.append(subagent.tag) - # Creating a dataframe for that agent and appending all the variable values - df = pd.DataFrame(columns=colmList) - for i in xList: - obj = root.xpath("xagent")[i-1].getchildren() - var=[] - for j in range(0,len(colmList)): - var.append(obj[j].text) - row = dict(zip(colmList,var)) - row_s = pd.Series(row) - row_s.name = os.path.splitext(os.path.basename(files))[0] - df = df.append(row_s) - del var,row,row_s - del colmList, xList - if df.empty == False: - return df - -# Function to generate data panels given the agent-names -def gendp(agtname): - oneinst = d[agtname] - dfa = pd.DataFrame() - for i in range(0,len(oneinst)): - dfa = dfa.append(oneinst[i][0]) - dfa = dfa.reset_index() - dfa.set_index(['index',dfa.index], inplace=True) - dp = dfa.to_panel() - del oneinst, dfa - pnl = dp.swapaxes(0,2, copy=True) - return pnl - - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='xml_hdf5.py', description='Generates HDF5 files from XML files. Creates one HDF5 file for all xml files in one particular folder') - parser.add_argument('modelpath', help='Path to model_xml folder containing all the subfolders and corresponding model.xml files', nargs=1, type=str) - parser.add_argument('model_file_name', help='Name of the main xml_model file', nargs=1, type=str) - parser.add_argument('datapath', help='Path to main data folder containing subfolders for all the runs, which contains XML files that require processing', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - args = parser.parse_args() - model = args.modelpath[0]+ '/' + args.model_file_name[0] - - # Check if the model xml file exists - if not file_exist(model): - error("- Model file (%s) does not exist" % model) - - # Set input parameters - input_xml_folder = '' - input_xml_folder = args.datapath[0] - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_xml_folder): - dir_list.append(dirpath) - if len(dir_list) == 0: - error("- Make sure the specified input directory is an actual directory, and not a file!") - if os.getcwd() == os.path.abspath(input_xml_folder): - error("- BAD IDEA!!! Execution script and input db files both inside a single folder <<" +os.getcwd()+">>. Might cause a recursive loop leading to erroneous output in some cases. Expected at least one level of separation. Please keep the script somewhere else and retry!") - - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - # Set output parameters - output_folder = '' - if args.outpath: - output_folder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - #output_folder = './output_'+os.path.basename(input_xml_folder) # Creates output folder in the same folder where Python script is located. - output_folder = os.path.dirname(input_xml_folder)+'/output_'+os.path.basename(input_xml_folder) # Creates output folder in the same folder where input folder is located - - - # Function call to check if the output folder already exists, and create if not present - dir_check(output_folder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print arg, - print - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - sys.stdout.write("\r" + arg) - sys.stdout.flush() - print - else: - statusprint = lambda *a: None - - # Parse model_xml file to determine required agents - verboseprint ("\n- Analysing model file\n") - models = [model] - alist =[] - while len(models) > 0: - fname = models.pop() - dirname = os.path.dirname(fname) - # load xml file - verboseprint (" + parsing %s" % fname) - try: - dom = minidom.parse(fname) - except IOError: - warn("Unable to read model file (%s). No such path exists." % fname) - continue - except ExpatError: - error("Invalid XML file (%s)" % fname) - - # detect nested models - nodes = dom.getElementsByTagName("model") - for node in nodes: - status = get_node_text(node.getElementsByTagName("enabled")[0]) - if status != "true": continue - # add nested model file to list of files - modelfile = get_node_text(node.getElementsByTagName("file")[0]) - models.append(os.path.join(dirname, modelfile)) - del(nodes) - - # detect agents - nodes = dom.getElementsByTagName("xagent") - for node in nodes: - # get agent name - aname = get_node_text(node.getElementsByTagName("name")[0]) - alist.append(aname) - del(nodes) - verboseprint ("finished parsing") - del(dom) - verboseprint ("\n- Parsing model file complete") - - # Process each folder in the input directory - processed_folders = 0 - statusprint('\n- Analysing input folders... total no. of input folders: '+ str(F)+'\n') - for i in range(N,len(dir_list)): - statusprint('- Started processing folder: '+os.path.basename(dir_list[i])) - # Populate the list with all xml file names in that input folder - xml_filenames = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+XML_SUFFIX)): - xml_filenames.append(fname) - statusprint('- Total number of files in current folder: '+ str(len(xml_filenames))) - - agentnames = list(set(alist)) - # Store each dataframes for a particular agent in a dict, with agentnames as dict keys - d = {} - for f in xml_filenames: - for a in agentnames: - if a in d: - if gendf(f, a) is not None: - d[a].append([gendf(f, a)]) - else: - if gendf(f, a) is not None: - d[a]=[[gendf(f, a)]] - - # Generate the datapanels for each agents and store them in a HDF5 file - agt = d.keys() - n = 0 - for f in agt: - w = agt[n] - # Setting up HDF5 file for storing output - verboseprint ('\n- Preparing HDF5 file for writing contents of input folder: '+os.path.basename(dir_list[i])) - fname_tostore = output_folder +'/'+os.path.basename(dir_list[i])+'.h5' - store = pd.HDFStore(fname_tostore) - ##store = pd.HDFStore(fname_tostore, 'w', complevel = 1, complib ='bzip2', fletcher32 = True) - store[f] = gendp(w) - store.close() - n = n+1 - # Get the progress status within a folder - percent = round((float(n)/len(agt))*100,2) - statusprint('- Number processed agents: '+str(n)+', of total: '+str(len(agt))+' Current Folder progress:'+ str(percent) +'%'), - verboseprint ('- Successfully closed HDF5 file for: '+os.path.basename(dir_list[i])) - # Get the overall progress status - statusprint('- Finished processing folder: '+ os.path.basename(dir_list[i])+'\n') - processed_folders = processed_folders + 1 - main_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Number processed folders: '+str(processed_folders)+', of total: '+str(F)+' Total progress:'+ str(main_percent) +'%\n'), - diff --git a/src/data_conversion_scripts/README.rst b/src/data_conversion_scripts/README.rst deleted file mode 100644 index e07dfc3..0000000 --- a/src/data_conversion_scripts/README.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. -*- mode: rst -*- - -Data Conversion Scripts -======================= - -This repository contains all the necessary Python scripts to convert between different data formats. - -The conversions that are possible are: - -- ``SQLite`` db format -> ``xml`` format -- ``SQLite`` db format -> ``hdf5`` format -- ``xml`` format -> ``hdf5`` format - -The repo also includes script to convert between different structures within the hdf5 file (``set_*_run_*_iters`` format to ``Agentname`` *[Eurostat, Bank etc]* ) format. - -Dependencies -============ - -The scripts in this repository are tested to work with both Python 2.7.10 / Python 3.5.2 and above. - -The required dependencies are : Pandas >= 0.19.2 , scipy >=0.18.1, numpy >= 1.11.3 - -Also, the data storage format requires the following dependency : pytables - -Usage -===== - -The detailed documentation on how to use the module can be found on the */doc* folder in this repo. diff --git a/src/data_conversion_scripts/src/db_hdf5/db_hdf5_v1.py b/src/data_conversion_scripts/db_hdf5/db_hdf5_v1.py similarity index 52% rename from src/data_conversion_scripts/src/db_hdf5/db_hdf5_v1.py rename to src/data_conversion_scripts/db_hdf5/db_hdf5_v1.py index 2b0cbc4..4bab66b 100644 --- a/src/data_conversion_scripts/src/db_hdf5/db_hdf5_v1.py +++ b/src/data_conversion_scripts/db_hdf5/db_hdf5_v1.py @@ -1,75 +1,85 @@ #!/usr/bin/env python3 ################################################################################################################# -## This script creates HDF5 files from corresponding SQLite db files in many-to-one fashion. So, for all db files -## present in one input folder, one equivalent HDF5 file is created in the output folder. For input folders -## containing more hierarchies, if run with a flag -r, one HDF5 file is created for contents of each subfolder. +## This script creates HDF5 files from corresponding SQLite db files in many-to-one fashion. So, for all db files +## present in one input folder, a single, combined HDF5 file is created in the output folder. For input folders +## containing multiple hierarchies, if the flag -r is set, a separate HDF5 file is created for the contents of each subfolder. ################################################################################################################# from __future__ import print_function import sqlite3, sys, glob, os, argparse, errno import pandas as pd +from pathlib import Path pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/ appended to' later on + # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/ appended to' later on DB_SUFFIX = '.db' -# Function to check for existing directories, and create a new one if not present +# Function to check for existing directories, and create a new one if not present def dir_check(d): if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - + outpath = os.path.abspath(d) + print("- Directory used for output: [" + outpath + "]") + else: os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") + outpath = os.path.abspath(d) + print("- Output directory created: [" + outpath + "]") # Function to print out the error messages,if any, and exit def error(mesg): - print(">>>>> (Error): %s" % mesg, file = sys.stderr) + print(">>>>> (Error): {0}".format(mesg), file = sys.stderr) sys.exit(1) - +# Function to read df from SQLite table for agent_name def agent_dataframe(agent_name): agent_name = agent_name.decode('utf-8') df = pd.read_sql(("SELECT * from "+ agent_name), con) - - # Ignoring the empty tables + #print(df.head()) + + # Ignoring the empty tables if df.empty == False: - #Converting the datatypes to make all of them uniform, to avoid pickling later on when writing to HDF - types = df.apply(lambda x: pd.lib.infer_dtype(x.values)) - + + #Converting the datatypes to make all of them uniform, to avoid pickling later on when writing to HDF + types = df.apply(lambda x: pd.api.types.infer_dtype(x.values)) + for col in types[types=='floating'].index: - df[col] = df[col].astype(str) - + df[col] = df[col].astype(str) + for col in types[types=='mixed'].index: df[col] = df[col].astype(str) - + for col in types[types=='unicode'].index: df[col] = df[col].astype(str) - - # Keeping count of the individual agent instance - IterationList =[] - agtin = str('agent_inst').encode('utf-8') - IterationList = df['_ITERATION_NO'].tolist() - j=0 - agentinstance_index =[j] - for i in range(1,len(IterationList)): - if IterationList[i] == IterationList[i-1]: - j=j+1 - agentinstance_index.append(j) - else: - j=0 - agentinstance_index.append(j) - # Adding the agent instance count as a separate column to the dataframe - df[agtin] = agentinstance_index - # Creating hierarchial index for df with the iteration number, and agent instance count - df.set_index(['_ITERATION_NO',agtin], inplace=True) + + # Rename col '_ITERATION_NO' to 'iter' + if '_ITERATION_NO' in types.index: + df.rename(columns={'_ITERATION_NO': 'iter'}, inplace=True) + + if 'id' in types.index: + #print("Have 'id' variable") + # Create hierarchical index for df with iteration, agent id + # Set [iters,id] as the new index + df.set_index(['iter','id'], inplace=True) + else: + #print("No 'id' variable stored in DB") + # Count of agent instances + IterationList =[] + agtin = str('agent_inst').encode('utf-8') + IterationList = df['iter'].tolist() + j=0 + agentinstance_index = [j] + for i in range(1,len(IterationList)): + if IterationList[i] == IterationList[i-1]: + j=j+1 + agentinstance_index.append(j) + else: + j=0 + agentinstance_index.append(j) + df[agtin] = agentinstance_index # Add agent instance count as separate column to dataframe + df.set_index(['iter', agtin], inplace=True) # Create hierarchical index for df with iteration and agent instance count + #print(df.head(10)) return df - -def gendp(dframes): - dp = dframes.to_panel() - #pnl = dp.swapaxes(0,2, copy=True) #This will allow to swap the datapanel axes, if needed - return dp def gen_hdf(fname): global con, cur, agent_name @@ -77,8 +87,8 @@ def gen_hdf(fname): try: con = sqlite3.connect(fname) cur = con.cursor() - - # Get the name of all the tables in the SQLite DB i.e. the agents + + # Get the name of all the tables in the SQLite DB i.e. the agent types cur.execute("SELECT name from sqlite_master WHERE type='table';") agent_list = [x[0].encode('ascii', 'ignore') for x in cur.fetchall()] w = str('_iters_').encode('utf-8') @@ -86,42 +96,73 @@ def gen_hdf(fname): agent_list.remove(w) # Iterate through all the agent tables in the DB and add each to a unique Dataframe for agent_name in agent_list: - verboseprint ('- Processing table values for: '+str(agent_name)) + verboseprint ('- Processing table values for: '+str(agent_name).replace("b'","'")) dfa = agent_dataframe(agent_name) - verboseprint ('- Obtained Dataframe for: '+str(agent_name)) + verboseprint ('- Obtained Dataframe for: '+str(agent_name).replace("b'","'")) # Checking and ignoring empty agent tables that might be in the DB if dfa is not None: verboseprint ('- Preparing HDF5 file for writing') a = agent_name.decode('utf-8') - s = str(os.path.basename(fname))[:-3] +'/'+str(a) - # Storing the panels in the HDF Store - store[s] = gendp(dfa) - verboseprint ('- Successfully wrote: '+str(agent_name)+ ' into HDF5 file \n') + s = str(os.path.basename(fname))[:-3] +'/'+str(a).replace("b'","'") + # Storing the object (dataframe) in the HDF Store + store[s] = dfa + verboseprint ('- Successfully wrote: '+str(agent_name).replace("b'","'")+ ' into HDF5 file. \n') except sqlite3.Error as e: print("Error ", e.args[0]) sys.exit(1) finally: if con: - con.close() + con.close() + #verboseprint ('- Successfully closed HDF5 file: '+os.path.basename(fname_tostore)+'\n') + #store.close() del dfa if __name__ == "__main__": # Setup for command line arguments - parser = argparse.ArgumentParser(prog='db_hdf5_v1.py', description='Converts the SQLite database files to HDF5 files. Combines multiple db files into a single HDF5 file.') + parser = argparse.ArgumentParser(prog='db_hdf5_v1.py', description='Converts SQLite database files to HDF5 files (many-to-one). The script combines multiple .db files into a single .h5 file.') parser.add_argument('dbpath', help='Path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') + parser.add_argument('-o', '--outpath', help='Path to output folder where the output where h5 files should be written', nargs=1, type=str) + parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') parser.add_argument('-r', '--recursive', help = 'Use recursion to process all subfolders within given folder',action='store_true') + parser.add_argument('-c', '--chunksize', help='Set the chunksize for HDF5 files', nargs='?', type=int, const=500, default=500, action='store', required=False) + parser.add_argument('-z', '--compress', help='Use internal compression in HDFStore', nargs='?', type=int, const=1, default=1, action='store', required=False) args = parser.parse_args() - + + #Setup for verbose arguments + if args.verbose: + def verboseprint(*args): + for arg in args: + print(arg, end=' ') + print() + else: + verboseprint = lambda *a: None + + #Setup for process status arguments + if args.status: + def statusprint(*args): + for arg in args: + print(arg, end=' ') + print() + else: + statusprint = lambda *a: None + + # Settings for HDF5 files + chunksize = args.chunksize + verboseprint ('- Chunksize used in HDFStore: ' + str(chunksize)) + h5_compression = args.compress + if (h5_compression): + verboseprint ('- Compression used in HDFStore: ' + str(h5_compression)) + else: + verboseprint ('- Compression used in HDFStore: ' + 'None') + # Set input parameters input_dbfolder = args.dbpath[0] - if os.getcwd() == os.path.abspath(input_dbfolder): - error("- Python script and data files not allowed in a single folder. Expects atleast a level of folder hierarchy. Fix issue and retry! ") + if os.getcwd() == os.path.abspath(input_dbfolder): + error("- Python script and data files not allowed in a single folder. Expects at least one level of folder hierarchy. Please fix the issue and retry.") dir_list =[] if args.recursive: @@ -131,70 +172,60 @@ def gen_hdf(fname): else: dir_list.append(os.path.abspath(input_dbfolder)) - # Set output parameters output_folder = '' if args.outpath: output_folder = args.outpath[0] else: # Choose one of the options below and comment out the other as desired. - - #output_folder = './output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Creates output folder in the same folder where Python script is located. - #output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Creates output folder in the same folder where input folder is located - output_folder = os.path.dirname(input_dbfolder) # Creates no output folder and places h5 file in same folder as input file - # Function call to check if the output folder already exists, and create if not present + #output_folder = './output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Create output folder in the same folder where Python script is located. + #output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Create output folder in the same folder where input folder is located. + output_folder = os.path.dirname(input_dbfolder) # Create no output folder and places h5 file in same folder as input file. + + # Function call to check if the output folder already exists, and create if not present dir_check(output_folder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print(arg, end=' ') - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - print(arg, end=' ') - print() - else: - statusprint = lambda *a: None - F = len(dir_list) + + F = len(dir_list) processed_folders = 0 statusprint('\n- Total number of folders: '+ str(F)+'\n') - + for d in dir_list: - statusprint('- Started processing folder: '+os.path.basename(d)) + statusprint('- Started processing folder: ' + os.path.basename(d)) + statusprint('- \t complete path: ' + os.path.abspath(d)) + # Populate the list with all sql file names in the folder - db_file_list = [] + db_file_list = [] for fname in glob.glob(os.path.join(d, '*'+DB_SUFFIX)): db_file_list.append(fname) + statusprint('- \t adding to list: '+str(fname)) statusprint('- Total number of files in current folder: '+ str(len(db_file_list))) - + # Setting up HDF5 file for storing output - fname_tostore = output_folder +'/'+os.path.basename(d)+'.h5' - verboseprint ('\n- Preparing HDF5 file for writing contents of input folder: '+os.path.basename(d)) - store = pd.HDFStore(fname_tostore, 'w', complevel = 1, complib ='bzip2', fletcher32 = True) - - # Generate panels + fname_tostore = output_folder +'/'+os.path.basename(d)+'.h5' + verboseprint ('\n- Preparing HDF5 file for writing contents of input folder: ' + os.path.basename(d)) + verboseprint ('\n- \tfilename to store: ' + fname_tostore) + + if (h5_compression): + verboseprint ('\n- \t HDFStore using: compression, with chunksize: ' + str(chunksize)) + store = pd.HDFStore(fname_tostore, 'w', chunksize = chunksize, complevel = 1, complib ='bzip2', fletcher32 = True) # store with compression + else: + verboseprint ('\n- \t HDFStore using: no compression') + store = pd.HDFStore(fname_tostore, 'w') # to store without compression + + # Generate HDFStore objects and store in h5 file processed_files =[] for fname in sorted(db_file_list): - verboseprint ('\n- Started processing: '+os.path.basename(fname)) - + verboseprint ('\n- Started processing: ' + os.path.basename(fname)) gen_hdf(fname) processed_files.append(fname) percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), - verboseprint ('- Successfully closed HDF5 file for: '+os.path.basename(d)) + statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress: '+ str(percent) +'%'), + store.close() + verboseprint ('- Successfully closed HDF5 file for: '+os.path.basename(d)) + statusprint('- Finished processing folder: '+ os.path.basename(d)+'\n') processed_folders = processed_folders+1 f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - - - + statusprint('- Total progress: '+ str(f_percent) +'%', '\n') diff --git a/src/data_conversion_scripts/src/db_hdf5/db_hdf5_v2.py b/src/data_conversion_scripts/db_hdf5/db_hdf5_v2.py similarity index 53% rename from src/data_conversion_scripts/src/db_hdf5/db_hdf5_v2.py rename to src/data_conversion_scripts/db_hdf5/db_hdf5_v2.py index 0c8fc98..44df96a 100644 --- a/src/data_conversion_scripts/src/db_hdf5/db_hdf5_v2.py +++ b/src/data_conversion_scripts/db_hdf5/db_hdf5_v2.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 ################################################################################################################# -## This script creates HDF5 files from corresponding SQLite db files in one-to-one fashion. So, for each db file -## present in the input folder, equivalent HDF5 file is created in the output folder. To combine db files into -## a single HDF5 file , use version 1 of script named db_hdf5_v1.py instead +## This script creates HDF5 files from corresponding SQLite db files in one-to-one fashion. So, for each db file +## present in the input folder, a corresponding HDF5 file is created in the output folder. To combine db files into +## a single HDF5 file, use version 1 of script named db_hdf5_v1.py instead. ################################################################################################################# from __future__ import print_function @@ -12,79 +12,95 @@ pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on + # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on DB_SUFFIX = '.db' - -# Function to check for existing directories, and create a new one if not present + +# Function to check for existing directories, and create a new one if not present def dir_check(d): if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - + outpath = os.path.abspath(d) + print("- Directory used for output: [" + outpath + "]") + else: os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") - + outpath = os.path.abspath(d) + print("- Output directory created: [" + outpath + "]") # Function to print out the error messages,if any, and exit def error(mesg): - print(">>>>> (Error): %s" % mesg, file = sys.stderr) + print('>>>>> (Error): {0}' .format(mesg), file = sys.stderr) sys.exit(1) - +# Function to read df from SQLite table for agent_name def agent_dataframe(agent_name): agent_name = agent_name.decode('utf-8') df = pd.read_sql(("SELECT * from "+ agent_name), con) - - # Ignoring the empty tables + #print(df.head()) + + # Ignoring the empty tables if df.empty == False: - #Converting the datatypes to make all of them uniform, to avoid pickling later on when writing to HDF - types = df.apply(lambda x: pd.lib.infer_dtype(x.values)) - + + #Converting the datatypes to make all of them uniform, to avoid pickling later on when writing to HDF + types = df.apply(lambda x: pd.api.types.infer_dtype(x.values)) + for col in types[types=='floating'].index: - df[col] = df[col].astype(str) - + df[col] = df[col].astype(str) + for col in types[types=='mixed'].index: df[col] = df[col].astype(str) - + for col in types[types=='unicode'].index: df[col] = df[col].astype(str) - - # Keeping count of the individual agent instance - IterationList =[] - agtin = str('agent_inst').encode('utf-8') - IterationList = df['_ITERATION_NO'].tolist() - j=0 - agentinstance_index =[j] - for i in range(1,len(IterationList)): - if IterationList[i] == IterationList[i-1]: - j=j+1 - agentinstance_index.append(j) - else: - j=0 - agentinstance_index.append(j) - # Adding the agent instance count as a separate column to the dataframe - df[agtin] = agentinstance_index - # Creating hierarchial index for df with the iteration number, and agent instance count - df.set_index(['_ITERATION_NO',agtin], inplace=True) + + # Rename col '_ITERATION_NO' to 'iter' + if '_ITERATION_NO' in types.index: + df.rename(columns={'_ITERATION_NO': 'iter'}, inplace=True) + + if 'id' in types.index: + #print("Have 'id' variable") + # Create hierarchical index for df with iteration, agent id + # Set [iters,id] as the new index + df.set_index(['iter','id'], inplace=True) + else: + #print("No 'id' variable stored in DB") + # Count of agent instances + IterationList =[] + agtin = str('agent_inst').encode('utf-8') + IterationList = df['iter'].tolist() + j=0 + agentinstance_index = [j] + for i in range(1,len(IterationList)): + if IterationList[i] == IterationList[i-1]: + j=j+1 + agentinstance_index.append(j) + else: + j=0 + agentinstance_index.append(j) + df[agtin] = agentinstance_index # Add agent instance count as separate column to dataframe + df.set_index(['iter', agtin], inplace=True) # Create hierarchical index for df with iteration and agent instance count + #print(df.head(10)) return df - -def gendp(dframes): - dp = dframes.to_panel() - #pnl = dp.swapaxes(0,2, copy=True) # This will allow to swap the datapanel axes, if needed - return dp -def gen_hdf(fname, output_folder): - outFileName = output_folder + '/' + os.path.basename(fname)[:-len(DB_SUFFIX)] +def gen_hdf(fname, output_folder, chunksize, h5_compression): + outFileName = output_folder + '/' + os.path.basename(fname)[:-len(DB_SUFFIX)] fname_tostore = outFileName +'.h5' #verboseprint ('- Preparing HDF5 file: '+os.path.basename(fname_tostore)+'for writing') - store = pd.HDFStore(fname_tostore, 'w', chunksize = 500, complevel = 1, complib ='bzip2', fletcher32 = True) + #store = pd.HDFStore(fname_tostore, 'w', chunksize = 500, complevel = 1, complib ='bzip2', fletcher32 = True) + + if (h5_compression): + verboseprint ('\n- \t HDFStore using: compression, with chunksize: ' + str(chunksize)) + store = pd.HDFStore(fname_tostore, 'w', chunksize = chunksize, complevel = 1, complib ='bzip2', fletcher32 = True) # store with compression + else: + verboseprint ('\n- \t HDFStore using: no compression') + store = pd.HDFStore(fname_tostore, 'w') # to store without compression + global con, cur, agent_name con = None try: con = sqlite3.connect(fname) cur = con.cursor() - - # Get the name of all the tables in the SQLite DB i.e. the agents + + # Get the name of all the tables in the SQLite DB i.e. the agent types cur.execute("SELECT name from sqlite_master WHERE type='table';") agent_list = [x[0].encode('ascii', 'ignore') for x in cur.fetchall()] w = str('_iters_').encode('utf-8') @@ -92,22 +108,22 @@ def gen_hdf(fname, output_folder): agent_list.remove(w) # Iterate through all the agent tables in the DB and add each to a unique Dataframe for agent_name in agent_list: - verboseprint ('- Processing table values for: '+str(agent_name)) + verboseprint ('- Processing table values for: '+str(agent_name).replace("b'","'")) dfa = agent_dataframe(agent_name) - verboseprint ('- Obtained Dataframe for: '+str(agent_name)) + verboseprint ('- Obtained Dataframe for: '+str(agent_name).replace("b'","'")) # Checking and ignoring empty agent tables that might be in the DB if dfa is not None: verboseprint ('- Preparing HDF5 file for writing') a = agent_name.decode('utf-8') - # Storing the panels in the HDF Store - store[a] = gendp(dfa) - verboseprint ('- Successfully wrote: '+str(agent_name)+ ' into HDF5 file. \n') + # Storing the object (dataframe) in the HDF Store + store[a] = dfa + verboseprint ('- Successfully wrote: '+str(agent_name).replace("b'","'")+ ' into HDF5 file. \n') except sqlite3.Error as e: print("Error ", e.args[0]) sys.exit(1) finally: if con: - con.close() + con.close() verboseprint ('- Successfully closed HDF5 file: '+os.path.basename(fname_tostore)+'\n') store.close() del dfa @@ -115,25 +131,55 @@ def gen_hdf(fname, output_folder): if __name__ == "__main__": # Setup for command line arguments - parser = argparse.ArgumentParser(prog='db_hdf5_v2.py', description='Converts the SQLite database files to HDF5 files. For each db files, creates an equivalent HDF5 file.') + parser = argparse.ArgumentParser(prog='db_hdf5_v2.py', description='Converts SQLite database files to HDF5 files (1-to-1). The script creates a corresponding .h5 file for each .db file.') parser.add_argument('dbpath', help='Path to folder containing the .db files', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') + parser.add_argument('-o', '--outpath', help='Path to output folder where h5 files should be written', nargs=1, type=str) + parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - parser.add_argument('-r', '--recursive', help = 'Use recursion to process all subfolders within given folder',action='store_true') + parser.add_argument('-r', '--recursive', help = 'Use recursion to process all subfolders',action='store_true') + parser.add_argument('-c', '--chunksize', help='Set the chunksize for HDF5 files', nargs='?', type=int, const=500, default=500, action='store', required=False) + parser.add_argument('-z', '--compress', help='Use internal compression in HDFStore', nargs='?', type=int, const=1, default=1, action='store', required=False) args = parser.parse_args() - + + #Setup for verbose arguments + if args.verbose: + def verboseprint(*args): + for arg in args: + print(arg, end=' ') + print() + else: + verboseprint = lambda *a: None + + #Setup for process status arguments + if args.status: + def statusprint(*args): + for arg in args: + print(arg, end=' ') + print() + else: + statusprint = lambda *a: None + + # Settings for HDF5 files + chunksize = args.chunksize + verboseprint ('- Chunksize used in HDFStore: ' + str(chunksize)) + h5_compression = args.compress + if (h5_compression): + verboseprint ('- Compression used in HDFStore: ' + str(h5_compression)) + else: + verboseprint ('- Compression used in HDFStore: ' + 'None') + + # Set input parameters input_dbfolder = args.dbpath[0] - if os.getcwd() == os.path.abspath(input_dbfolder): - error("- Python script and data files not allowed in a single folder. Expects atleast a level of folder hierarchy. Fix issue and retry! ") + if os.getcwd() == os.path.abspath(input_dbfolder): + error("- Python script and data files not allowed in a single folder. The script expects at least one level of folder hierarchy. Please fix the issue and retry.") dir_list =[] if args.recursive: # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): + for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): dir_list.append(os.path.abspath(dirpath)) else: dir_list.append(os.path.abspath(input_dbfolder)) @@ -144,57 +190,38 @@ def gen_hdf(fname, output_folder): output_folder = args.outpath[0] else: # Choose one of the options below and comment out the other as desired. - #output_folder = './output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Creates output folder in the same folder where Python script is located. - #output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Creates output folder in the same folder where input folder is located - output_folder = os.path.dirname(input_dbfolder) # Creates no output folder and places h5 file in same folder as input file - - # Function call to check if the output folder already exists, and create if not present + #output_folder = './output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Create output folder in the same folder where Python script is located. + #output_folder = os.path.dirname(input_dbfolder)+'/output_'+os.path.basename(os.path.dirname(input_dbfolder)) # Create output folder in the same folder where input folder is located. + output_folder = os.path.dirname(input_dbfolder) # Create no output folder and places h5 file in same folder as input file. + + # Function call to check if the output folder already exists, and create if not present dir_check(output_folder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print(arg, end=' ') - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - print(arg, end=' ') - print() - else: - statusprint = lambda *a: None - + F = len(dir_list) - # Process each folder in the input directory + # Process each folder in the input directory processed_folders = 0 statusprint('\n- Total number of folders: '+ str(F)+'\n') for d in dir_list: - statusprint('- Started processing folder: '+os.path.basename(d)) + statusprint('- Started processing folder: '+os.path.basename(d)) + statusprint('- \t complete path: ' + os.path.abspath(d)) + # Populate the list with all sql file names in the folder db_file_list = [] for fname in glob.glob(os.path.join(d, '*'+DB_SUFFIX)): db_file_list.append(fname) - statusprint('- Total number of files within folder: '+ str(len(db_file_list))) - # Generate panels + statusprint('- \t adding to list: '+str(fname)) + statusprint('- Total number of files within folder: '+ str(len(db_file_list))) + + # Generate HDFStore objects and store in h5 file processed_files =[] for fname in sorted(db_file_list): verboseprint('\n- Started processing: '+os.path.basename(fname)) - - gen_hdf(fname, output_folder) + gen_hdf(fname, output_folder, chunksize, h5_compression) processed_files.append(fname) percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), + statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress: '+ str(percent) +'%'), statusprint('- Finished processing folder: '+os.path.basename(d)+'\n') processed_folders = processed_folders+1 f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - - - - + statusprint('- Total progress: '+ str(f_percent) +'%', '\n') diff --git a/src/data_conversion_scripts/src/db_xml/genxml.py b/src/data_conversion_scripts/db_xml/genxml.py similarity index 100% rename from src/data_conversion_scripts/src/db_xml/genxml.py rename to src/data_conversion_scripts/db_xml/genxml.py diff --git a/src/data_conversion_scripts/doc/usage_help.rst b/src/data_conversion_scripts/doc/usage_help.rst deleted file mode 100644 index 0521e97..0000000 --- a/src/data_conversion_scripts/doc/usage_help.rst +++ /dev/null @@ -1,80 +0,0 @@ -Notes on usage of the python scripts ------------------------------------- - -1. DB_HDF5 -~~~~~~~~~~ - -- These scripts convert ``SQLite`` database files to ``HDF5`` files. -- There are two versions of the script currently, each having a slightly different functionality: - -#. The script ``db_hdf5_v1.py`` creates HDF5 files from corresponding SQLite db files in *many-to-one* fashion. - So, for all db files present in one input folder, one equivalent HDF5 file is created in the output folder. - For input folders containing more hierarchies, a **-r** flag can be passed which results in one HDF5 file for the contents of each subfolder. - -#. The script ``db_hdf5_v2.py`` creates HDF5 files from corresponding SQLite db files in *one-to-one* fashion. - So, for each db file present in the input folder, an equivalent HDF5 file is created in the output folder. - To combine db files into a single HDF5 file , version 1 of script named ``db_hdf5_v1.py`` can be used instead. - -- The script runs on both Python 2.7.10 / Python 3.5.2 and above. Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: - - python db_hdf5_v1.py -h - -- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. -- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. - These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out - the option that is not preferred. Currently, the default is such that the output is created in the input folder. - -**Note:** *There is no difference in the size of combined HDF5 (many-to-one generated by db_hdf5_v1.py) and separate HDF5 (one-to-one generated by db_hdf5_v2.py) files, even when compressing.* - *However, separate HDF5 files are easier to manipulate when restructuring the HDF5 into files per Agent-type (i.e. Bank.h5, Eurostat.h5 and so on).* - *Therefore, use of db_hdf5_v2.py is recommended for simplicity.* - - -2. XML_HDF5 -~~~~~~~~~~~ - -- This script converts ``XML`` files to ``HDF5`` files. - -- This script creates HDF5 files from corresponding XML files in *many-to-one* fashion. So, for all xml files present in one input folder, one equivalent HDF5 file is created in the output folder. - For input folders containing more hierarchies, given that a *-r* flag is passed, one HDF5 file is created for contents of each subfolder. - -- The script runs on both Python 2.7.10 / Python 3.5.2 and above. Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: - - python xml_hdf5.py -h - -- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. -- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. - These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out - the option that is not preferred. Currently, the default is such that the output is created in the input folder. - - -3. DB_XML -~~~~~~~~~ - -- This script converts ``SQLite`` Db files to ``XML`` files. -- This script creates XML files from corresponding DB files in *one-to-many* fashion. For each DB file, a folder is created and multiple corresponding XML files are created inside the folder, based on time samples. -- The script runs on both Python 2.7.10 / Python 3.5.2 and above. Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: - - python genxml.py -h - -- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. -- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. - These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out - the option that is not preferred. Currently, the default is such that the output is created in the input folder. - - -4. MERGE_HDF_AGENTWISE -~~~~~~~~~~~~~~~~~~~~~~ - -- This script converts HDF5 files from ``set_*_run_*_iters.h5`` format to ``HDF5`` files per agent (**Eurostat.h5**, **Firm.h5** etc). -- So, for each HDF5 file present in the input folder, all corresponding agents are filtered and placed on a new agent-based HDF5 file. -- The script runs on both Python 2.7.10 / Python 3.5.2 and above. Any help regarding the execution of the script can be obtained by using the following command in any terminal window:: - - python merge_hdf_agentwise.py -h - -- There is an option to specify a desired output folder for the output files, by passing an **-o** flag. -- However, by default the output is created either in the folder containing the script, in the same folder that has the input folder, or the input folder itself. - These three options can be adjusted by commenting out the appropriate line in the script. Please look for the section *"Set output parameters"* in the script and comment out - the option that is not preferred. Currently, the default is such that the output is created in the input folder. - - - diff --git a/src/data_conversion_scripts/hdf_agentwise/merge_hdf_agentwise.py b/src/data_conversion_scripts/hdf_agentwise/merge_hdf_agentwise.py new file mode 100755 index 0000000..415595d --- /dev/null +++ b/src/data_conversion_scripts/hdf_agentwise/merge_hdf_agentwise.py @@ -0,0 +1,187 @@ + +#!/usr/bin/env python3 + +####################################################################################################################### +## This script converts HDF5 files from set_*_run_*_iters.h5 format to HDF5 files per agent (Eurostat.h5, Firm.h5 etc). +## So, for each HDF5 file present in the input folder, all corresponding agents are filtered and placed in new +## agent-based HDF5 file. +####################################################################################################################### + +from __future__ import print_function +import sqlite3, sys, glob, os, argparse, errno +import pandas as pd + +pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format + # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on + # We require the 'table' format here, due to the append in the function write_hdf() below. +DB_SUFFIX = '.h5' + +# Assumed input: a text file agentlist.txt containing the names of all agent types to process +# agentlist = ["Eurostat", "Government"] # Enter all agent-names to be processed + +def write_hdf(fname): + set_run_name = os.path.splitext(os.path.basename(fname))[0] + + print('\t\tTrying to open HDFStore in file: ' + os.path.basename(fname)) + print('\t\t' + set_run_name) + + # Read the input HDF5 file in memory + #store = pd.io.pytables.HDFStore(fname, mode='r') # this opens the file fname and gives a ref to it + store = pd.HDFStore(fname, mode='r') + + print('\t\tHDFStore now opened') + + # Filtering out each agent types from the input HDF5 file + print('\t\tstore.keys():') + print(store.keys()) + for key in store.keys(): + print('\t\tExtracting data for: ' + key.replace("b'","'")) + if agentname in key: + frame = store.select(key) # Retrieve pandas object stored in file, with optional WHERE selections + store_out[set_run_name] = frame # store the object as data group '/set_run_name' in the HDFStore store_out + store.close() + print('\t\tHDFStore closed.') + +# Function to check for existing directories, and create a new one if not present +def dir_check(d): + if os.path.exists(d): + outpath = os.path.abspath(d) + print("- Directory used for output: [" + outpath + "]") + + else: + os.makedirs(d) + outpath = os.path.abspath(d) + print("- Output directory created: [" + outpath + "]") + +# Function to print out the error messages,if any, and exit +def error(mesg): + print(">>>>> (Error): %s" % mesg, file = sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + # Setup for command line arguments + parser = argparse.ArgumentParser(prog='merge_hdf_agentwise.py', description='\ + * Converts HDF5 files containing multiple agent types to HDF5 files per agent type. \ + * Input can be a path to a folder that contains multiple HDF5 files, or to a folder hierarchy. \ + * Output will be to a user-specified folder. \ + * If the input folder is a folder hierarchy, it will be traversed, and a similar folder hierarchy will be created in the output folder.\ + * The contents of the agent-wise HDF5 files combines the data per agent type, as found in the traversed folders.') + parser.add_argument('hdfpath', help='Path to folder containing the individual hdf files', nargs=1, type=str) + parser.add_argument('agentlist', help='File containing name of agent-types to process', nargs=1, type=str) + parser.add_argument('-o', '--outpath', help='Path to the folder where the output will be stored', nargs=1, type=str) + parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') + parser.add_argument('-s', '--status', help='Get the overall progress of the process (status is less verbose)', action='store_true') + parser.add_argument('-c', '--chunksize', help='Set the chunksize for HDF5 files', nargs='?', type=int, const=500, default=500, action='store', required=False) + parser.add_argument('-z', '--compress', help='Use the internal compression (currently set to: bzip) in the HDFStore', nargs='?', type=int, const=1, default=1, action='store', required=False) + + args = parser.parse_args() + + #Setup for verbose arguments + if args.verbose: + def verboseprint(*args): + for arg in args: + print (arg, end=' ') + print() + else: + verboseprint = lambda *a: None + + #Setup for process status arguments + if args.status: + def statusprint(*args): + for arg in args: + print (arg, end=' ') + print() + else: + statusprint = lambda *a: None + + # Settings for HDF5 files + chunksize = args.chunksize + verboseprint ('- Chunksize used in HDFStore: ' + str(chunksize)) + h5_compression = args.compress + if (h5_compression): + verboseprint ('- Compression used in HDFStore: ' + str(h5_compression)) + else: + verboseprint ('- Compression used in HDFStore: ' + 'None') + + f_agentnames = args.agentlist[0] + agentlist = [] + with open(f_agentnames, 'r') as f_in: + for line in f_in: + agentlist.append(line.strip()) + + # Set input parameters + input_dbfolder = args.hdfpath[0] + + if os.getcwd() == os.path.abspath(input_dbfolder): + error("- Python script and data files not allowed in a single folder. Expects at least one level of folder hierarchy. Please fix the issue and retry.") + + dir_list =[] + # Checking for nested subdirectories within a directory + for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): + dir_list.append(dirpath) + if len(dir_list)>1: + N = 1 + F = len(dir_list)-1 + else: + N = 0 + F = len(dir_list) + + # Set output parameters + targetFolder = '' + if args.outpath: + targetFolder = args.outpath[0] + else: + # Choose one of the options below and comment out the other as desired. + #targetFolder = './merged_agent_wise' # Set output folder to the same folder where this script is located. + targetFolder = os.path.abspath(os.path.join(os.path.normpath(input_dbfolder), os.pardir))+'/merged_agent_wise' # Set output folder in the same parent folder as input folder. + #targetFolder = os.path.dirname(input_dbfolder) # Creates no output folder and places the merged_agent_wise h5 file in the parent folder of the input folder. + + # Function call to check if the output folder already exists, and create if not present + dir_check(targetFolder) + + # Process each folder in the input directory + processed_folders = 0 + statusprint('\n- Total number of folders: '+ str(F)+'\n') + for agentname in agentlist: # looping through each agent (memory conserving option) + for i in range(N,len(dir_list)): + # length of path until /last_folder + n = len(os.path.normpath(input_dbfolder)) + statusprint('- Processing files for agent-type: '+str(agentname)+'\n') + statusprint('- Started processing folder: '+os.path.basename(os.path.abspath(dir_list[i]))) + if N == 1: + output_folder = targetFolder + dir_list[i][n:] + dir_check(output_folder) + else: + output_folder = targetFolder + + # Populate the list with all sql file names in the folder + db_file_list = [] + for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): + db_file_list.append(fname) + statusprint('- adding to list: '+str(fname)) + statusprint('- Total number of files within folder: '+ str(len(db_file_list))) + + # Generate HDFStore objects and store in h5 file + processed_files =[] + + # Filename to write the new HDFStore + fname_tostore = output_folder + '/' + agentname +'.h5' + + if (h5_compression): + store_out = pd.HDFStore(fname_tostore, 'w', chunksize = chunksize, complevel = 1, complib ='bzip2', fletcher32 = True) # store with compression + else: + store_out = pd.HDFStore(fname_tostore, 'w') # to store without compression + + # Loop: process all h5 files set_*_run_*_iters.h5 and store them as groups in the new HDFStore 'store_out', written to file: fname_tostore + for fname in sorted(db_file_list): + verboseprint('\n- Started processing: '+os.path.basename(fname)) + write_hdf(fname) # write the pandas object (DataFrame) to the HDFStore + processed_files.append(fname) + percent = round((float(len(processed_files))/len(db_file_list))*100,2) + statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), + store_out.close() + statusprint('- Finished processing folder: '+os.path.basename(os.path.abspath(dir_list[i]))+'\n') + processed_folders = processed_folders+1 + if F > 1: + f_percent = round((float(processed_folders)/F)*100,2) + statusprint('- Total progress:'+ str(f_percent) +'%', '\n') diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/readme.txt b/src/data_conversion_scripts/misc/replace_xml_functions/test_example/readme.txt deleted file mode 100644 index 68f4d01..0000000 --- a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/readme.txt +++ /dev/null @@ -1,34 +0,0 @@ -# python script replace_xml_functions.py -# Sander vander Hoog, ETACE, Nbielefeld Univiersit -# 9 March 2017 - -TEST EXAMPLES - -These test examples are for testing the functionality of the script replace_xml_functions.py. - -How to run: -$ python replace_xml_functions.py [xml.c file] [shadow_xml.c file] [SHADOW_MODEL folder] [shadow_model.xml file] [-v] - -Arguments: -file1 = Name of original xml.c file, containing full code for the functions writeout_Agentname_agent() -file2 = Name of shadow_xml.c file, containing replacement code for the functions writeout_Agentname_agent() -modelpath = Path to the folder of shadow model xml -shadow model file = Name of the shadow model xml file - -Options: --v, --verbose = Get the status of the intermediate processing steps - -Example run commands: - -- If shadow model is in the current folder, and the script is also run from the current folder: -$ python replace_xml_functions.py xml.c shadow_xml.c . shadow_model.xml -v >out.txt - -- If the files are in separate input folders: -xml.c: inputs/REAL_MODEL -shadow_xml.c: inputs/XML -shadow_model.xml: inputs/SHADOW_MODEL, -then use: - -$ python replace_xml_functions.py inputs/REAL_MODEL/xml.c inputs/XML/shadow_xml.c inputs/SHADOW_MODEL shadow_model.xml -v >out.txt - -Output files are in output/. \ No newline at end of file diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/replace_xml_functions.py b/src/data_conversion_scripts/misc/replace_xml_functions/test_example/replace_xml_functions.py deleted file mode 100644 index 8b11022..0000000 --- a/src/data_conversion_scripts/misc/replace_xml_functions/test_example/replace_xml_functions.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python - -############################################################################################################### -# Replaces specified function of the input file (main) with the same function from another input file (shadow), -# and writes back to the main input file -# -# Usage: python replace_functions.py file1_name file2_name -# file1_name = main file, file2_name = shadow file -# -############################################################################################################### - -import os, sys, re, argparse, glob, errno -import lxml.etree as ET -from xml.dom import minidom -from xml.parsers.expat import ExpatError -from glob import glob as g - -if len(sys.argv) < 4: # specify both filenames xml.c and shawdow_xml.c, the path and file name of the shadow_model.xml file - print "replace_xml_functions.py:" - print " Replaces C functions in input file1 by functions with the same name in file2" - print "" - print "Usage: python replace_xml_functions.py [original xml.c file] [replacement xml.c file] [path to shadow model folder] [shadow model xml file]" - print "Example: python replace_xml_functions.py xml.c shadow_xml.c path_to_shadow_model shadow_model.xml" - print "" - sys.exit(1) - -# list of the functions to replace, currently a dummy list. A parser can be used to fill in this list from some other file (xml, txt), or the list can be added manually -#list_functions = ["write_Eurostat_agent", "write_Firm_agent" ] # names may have starting delimiter "{", but not strictly necessary - -def file_exist(fname): - f = g(fname) - if fname in f: return True - else: return False - -def get_node_text(node): - if len(node.childNodes) != 1 or node.childNodes[0].nodeType != node.TEXT_NODE: - error("getText() routine used on non-text XML node") - return node.childNodes[0].data.strip() - -def process_string( string_in, element ): - def find_between( s, first, last ): # function to find the string between two supplied limits - try: - start = s.index( first ) + len( first ) - end = s.index( last, start ) - return s[start:end] - except ValueError: - return "" - string_out = find_between(string_in,element,"}") # currently, the first limit (including upto first delimiter {) is read from the list of functions above, and the last limit is simply "}". can be replaced as needed - return string_out - -def ProcessFile(element): - f_a = open(sys.argv[1], 'r') - input_lines_a = f_a.read() - - #verboseprint ("Lines to replace:") - #verboseprint (process_string( input_lines_a, element )) - - f_a.close() - - f_b = open(sys.argv[2], 'r') - input_lines_b = f_b.read() - - #verboseprint ("Lines to paste:") - #verboseprint (process_string( input_lines_b , element )) - - f_b.close() - - # Replace the target string - replaced_lines_a = input_lines_a.replace(process_string(input_lines_a, element), process_string( input_lines_b, element )) - - f_out = open(sys.argv[1], 'w') - f_out.write(replaced_lines_a) - f_out.close() - -if __name__ == "__main__": - - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='replace_xml_functions.py', description='Replaces C functions in input file1 by functions with the same name in file2') - parser.add_argument('file1', help='Name of original xml.c file, containing full code for the functions writeout_Agentname_agent()', nargs=1, type=str) - parser.add_argument('file2', help='Name of shadow_xml.c file, containing replacement code for the functions writeout_Agentname_agent()', nargs=1, type=str) - parser.add_argument('modelpath', help='Path to the folder of shadow model xml', nargs=1, type=str) - parser.add_argument('shadowmodelfile', help='Name of the shadow model xml file', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - args = parser.parse_args() - model = args.modelpath[0]+ '/' + args.shadowmodelfile[0] - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print arg, - print - else: - verboseprint = lambda *a: None - - verboseprint ("Running replacements for XML writeout functions in xml.c") - - # Check if the model xml file exists - if not file_exist(model): - error("- Model file (%s) does not exist" % model) - else: - verboseprint("- Found model file (%s)" % model) - - # Parse model_xml file to determine required agents - verboseprint ("\n- Analysing model file\n") - models = [model] - list_functions =[] - while len(models) > 0: - fname = models.pop() - dirname = os.path.dirname(fname) - # load xml file - verboseprint (" + parsing %s" % fname) - try: - dom = minidom.parse(fname) - except IOError: - warn("Unable to read model file (%s). No such path exists." % fname) - continue - except ExpatError: - error("Invalid XML file (%s)" % fname) - - # detect nested models - nodes = dom.getElementsByTagName("model") - for node in nodes: - status = get_node_text(node.getElementsByTagName("enabled")[0]) - if status != "true": continue - # add nested model file to list of files - modelfile = get_node_text(node.getElementsByTagName("file")[0]) - models.append(os.path.join(dirname, modelfile)) - del(nodes) - - # detect agents - nodes = dom.getElementsByTagName("xagent") - for node in nodes: - # get agent name - aname = get_node_text(node.getElementsByTagName("name")[0]) - list_functions.append("write_"+aname+"_agent") - del(nodes) - del(dom) - verboseprint ("- Parsing model file complete") - verboseprint ("- Replacement list:") - for item in list_functions: - print (" "+item) - - #Processing per agent type - for element in list_functions: - ProcessFile(element) - - verboseprint ("Finished replacements") diff --git a/src/data_conversion_scripts/src/hdf_agentwise/merge_hdf_agentwise.py b/src/data_conversion_scripts/src/hdf_agentwise/merge_hdf_agentwise.py deleted file mode 100755 index 599b8fa..0000000 --- a/src/data_conversion_scripts/src/hdf_agentwise/merge_hdf_agentwise.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -####################################################################################################################### -## This script converts HDF5 files from set_*_run_*_iters.h5 format to HDF5 files per agent (Eurostat.h5, Firm.h5 etc). -## So, for each HDF5 file present in the input folder, all corresponding agents are filtered and placed in new -## agent-based HDF5 file. -####################################################################################################################### - -from __future__ import print_function -import sqlite3, sys, glob, os, argparse, errno -import pandas as pd - -pd.set_option('io.hdf.default_format','table') # Commenting this line out will write HDF5 as a fixed format, and not as a table format - # Writing as a fixed format is faster than writing as a table, but the file cannot be 'modified/appended to' later on - -DB_SUFFIX = '.h5' -#agentlist = ["Eurostat", "Government"] # Enter all agent-names that is to be processed - -def write_hdf(fname): - set_run_name = os.path.splitext(os.path.basename(fname))[0] - # Read the input HDF5 file in memory - store = pd.io.pytables.HDFStore(fname) - # Filtering out each agent types from the input HDF5 file - for key in store.keys(): - if agentname in key: - frame = store.select(key) - store_out[set_run_name] = frame - store.close() - -# Function to check for existing directories, and create a new one if not present -def dir_check(d): - if os.path.exists(d): - print("- Directory ["+os.path.basename(d)+ "] is used for output files") - else: - os.makedirs(d) - print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") - -# Function to print out the error messages,if any, and exit -def error(mesg): - print(">>>>> (Error): %s" % mesg, file = sys.stderr) - sys.exit(1) - -if __name__ == "__main__": - # Setup for command line arguments - parser = argparse.ArgumentParser(prog='merge_hdf_agentwise.py', description='Converts the SQLite database files to HDF5 files. For each db files, creates an equivalent HDF5 file.') - parser.add_argument('hdfpath', help='Path to folder containing the individual hdf files', nargs=1, type=str) - parser.add_argument('agentlist', help='File containing name of agent-types to process', nargs=1, type=str) - parser.add_argument('-o', '--outpath', help='Path to the folder where the output is desired', nargs=1, type=str) - parser.add_argument('-v', '--verbose', help='Get the status of the intermediate processing steps', action='store_true') - parser.add_argument('-s', '--status', help='Get the total progress of the processing', action='store_true') - - args = parser.parse_args() - - f_agentnames = args.agentlist[0] - agentlist = [] - with open(f_agentnames, 'r') as f_in: - for line in f_in: - agentlist.append(line.strip()) - - # Set input parameters - input_dbfolder = args.hdfpath[0] - - if os.getcwd() == os.path.abspath(input_dbfolder): - error("- Python script and data files not allowed in a single folder. Expects atleast a level of folder hierarchy. Fix issue and retry! ") - - dir_list =[] - # Checking for nested subdirectories within a directory - for (dirpath,dirnames,filenames) in os.walk(input_dbfolder): - dir_list.append(dirpath) - if len(dir_list)>1: - N = 1 - F = len(dir_list)-1 - else: - N = 0 - F = len(dir_list) - - # Set output parameters - targetFolder = '' - if args.outpath: - targetFolder = args.outpath[0] - else: - # Choose one of the options below and comment out the other as desired. - - targetFolder = './output_'+os.path.basename(os.path.normpath(input_dbfolder)) # For output folder in the same folder where Python script is located. - #targetFolder = os.path.abspath(os.path.join(os.path.normpath(input_dbfolder), os.pardir))+'/output_'+os.path.basename(os.path.normpath(input_dbfolder)) # For output folder in the same folder where input folder is located - #targetFolder = os.path.dirname(input_dbfolder) # Creates no output folder and places h5 file in same folder as input file - - - # Function call to check if the output folder already exists, and create if not present - dir_check(targetFolder) - - #Setup for verbose arguments - if args.verbose: - def verboseprint(*args): - for arg in args: - print (arg, end=' ') - print() - else: - verboseprint = lambda *a: None - - #Setup for process status arguments - if args.status: - def statusprint(*args): - for arg in args: - print (arg, end=' ') - print() - else: - statusprint = lambda *a: None - - # Process each folder in the input directory - processed_folders = 0 - statusprint('\n- Total number of folders: '+ str(F)+'\n') - for agentname in agentlist: # looping through each agent (memory conserving option) - for i in range(N,len(dir_list)): - n = len(os.path.normpath(input_dbfolder)) + 2 - statusprint('- Processing files for agent-type: '+str(agentname)+'\n') - statusprint('- Started processing folder: '+os.path.basename(os.path.abspath(dir_list[i]))) - if N == 1: - output_folder = targetFolder + dir_list[i][n:] - dir_check(output_folder) - else: - output_folder = targetFolder - - # Populate the list with all sql file names in the folder - db_file_list = [] - for fname in glob.glob(os.path.join(dir_list[i], '*'+DB_SUFFIX)): - db_file_list.append(fname) - statusprint('- Total number of files within folder: '+ str(len(db_file_list))) - # Generate panels - processed_files =[] - #print (db_file_list) - outFileName = output_folder + '/' + agentname +'.h5' - #store_out = pd.HDFStore(outFileName, 'w') # to store without compression - store_out = pd.HDFStore(outFileName, 'w', chunksize = 500, complevel = 1, complib ='bzip2', fletcher32 = True) # store with compression - for fname in sorted(db_file_list): - verboseprint('\n- Started processing: '+os.path.basename(fname)) - write_hdf(fname) - processed_files.append(fname) - percent = round((float(len(processed_files))/len(db_file_list))*100,2) - statusprint('- Number processed files: '+str(len(processed_files))+', of total: '+str(len(db_file_list))+' Progress:'+ str(percent) +'%'), - store_out.close() - statusprint('- Finished processing folder: '+os.path.basename(os.path.abspath(dir_list[i]))+'\n') - processed_folders = processed_folders+1 - if F > 1: - f_percent = round((float(processed_folders)/F)*100,2) - statusprint('- Total progress:'+ str(f_percent) +'%', '\n') - diff --git a/src/data_conversion_scripts/xml_db/gendb.py b/src/data_conversion_scripts/xml_db/gendb.py new file mode 100755 index 0000000..2beb5bd --- /dev/null +++ b/src/data_conversion_scripts/xml_db/gendb.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# $Id: gendb.py 2145 2009-09-23 08:51:01Z lsc $ +# +# Copyright (c) 2009 STFC Rutherford Appleton Laboratory +# Author: Lee-Shawn Chin +# Date : July 2009 +# +# NOTE: This is a proof-of-concept implementation. It was not designed for +# production use but merely to explore what is possible +# Use at your own risk ;) +# +import argparse, os, re, sys +from xml.dom import minidom +from xml.parsers.expat import ExpatError +from glob import glob +import sqlite3 + +outfile = "iters.db" + +def error(mesg): + #print >>sys.stderr, ">>>>> (Error): {0}".format(mesg) + print(">>>>> (Error): {0}".format(mesg), file=sys.stderr) + sys.exit(1) + +def warn(mesg): + #print >>sys.stderr, ">> (Warning): {0}".format(mesg) + print(">> (Warning): {0}".format(mesg), file=sys.stderr) + +def file_exist(fname): + f = glob(fname) + if fname in f: return True + else: return False + +def get_node_text(node): + if len(node.childNodes) != 1 or node.childNodes[0].nodeType != node.TEXT_NODE: + error("getText() routine used on non-text XML node") + return node.childNodes[0].data.strip() + +def get_agent_sql(data): + # "data" should be in format + # data["name"] = "agent_name" + # data["iter"] = 1 + # data["vars"] = {} + # data["vars"]["varname"] = value + global agents + + try: + name = data["name"] + vars = data["vars"] + iter = data["iter"] + except: + error("BUG!! contact script author") + + # check agent exists + if name not in agents.keys(): + error("Unknown agent found: {0}".format(name)) + + # retrieve values in the right order + keys = [] + values = [] + for v in vars.keys(): + if v == "name": continue + if v not in agents[name]: + error("Unknown agent memory : {0}.{1}".format(name, v)) + keys.append(v) + values.append(vars[v]) + + sql = [] + sql.append("insert into {0}".format(name)) + sql.append("(_ITERATION_NO,{0})".format(",".join(keys))) + sql.append("values ({0},'{1}');" .format(iter, "','".join(values))) + return " ".join(sql) + +def load_iteration_file(itno, basedir, db): + + filename = os.path.join(basedir, "{0}.xml".format(itno)) + print(" + Reading file {0}".format(filename)) + + if not file_exist(filename): + error("Strange! Input file disappeared : {0}".format(filename)) + try: + file = open(filename, "r") + except: + error("Could not open file {0}".format(filename)) + + + db.execute("INSERT INTO _iters_(itno) VALUES ({0});".format(i)) + + data = {} + in_xagent_tag = False + line = file.readline() + re_tag = re.compile(r"<(\w+)>([^<]*)") + + while line: + line = line.strip() + + if not line: # ignore empty lines + # get next line and restart loop + line = file.readline() + continue + + if in_xagent_tag: + if line == "": # end if this agent def + in_xagent_tag = False + db.execute(get_agent_sql(data)) # store data in db + del(data) + + else: # next agent mem + match = re_tag.match(line) + if not match: error("Unknown line: {0}".format(line)) + + tag = match.group(1) + val = match.group(2) + + if tag == "name": + data["name"] = val + else: + data["vars"][tag] = val + + else: # outside xagent tag + if line == "": + in_xagent_tag = True + data = {} + data["iter"] = i + data["vars"] = {} + + + line = file.readline() # get next line + + file.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog='gendb.py', description='Script creates SQLite db file from corresponding XML files in many-to-one fashion, and skips missing Agents.') + parser.add_argument('--model', '-m', help='Set (absolute or relative) path to the directory that contains the model, including the name of the model file (model.xml).', nargs=1, type=str, required=True) + parser.add_argument('--xml', '-x', help='Set (absolute or relative) path to the directory that contains the data to be converted (xml files).', nargs=1, type=str, required=True) + parser.add_argument('--output', '-o', help='Set (absolute or relative) path to the directory where the output file should be written (iters.db). If the folder does not exist, it will be created.', nargs=1, type=str, required=False) + args = parser.parse_args() + + + # check if output argument is used + if args.output: + # check if a custom .db file is used + if args.output[0].rsplit('/', 1)[1].endswith('.db'): + # check if output directory exists + if not os.path.isdir(args.output[0].rsplit('/', 1)[0]): + error("Output directory ({0}) does not exist".format(args.output[0].rsplit('/', 1)[0])) + else: + outfile = str(args.output[0]) + else: + # check if output directory exists + if not os.path.isdir(args.output[0]): + error("Output directory ({0}) does not exist".format(args.output[0])) + else: + # check for output path and add an ending slash if necessary + if args.output[0][-1:] is '/': + outfile = str(args.output[0] + outfile) + else: + outfile = str(args.output[0] + '/' + outfile) + + # get input arguments + model = args.model[0] + itsdir = args.xml[0] + + # check that model file exists + if not file_exist(model): + error("Model file ({0}) does not exist".format(model)) + + # ensure directory exists + if not os.path.isdir(itsdir): + error("Input directory ({0}) does not exist".format(itsdir)) + + # Retrieve list of data files + datafiles = glob(os.path.join(itsdir, "[0-9]*.xml")) + if len(datafiles) == 0: + error("No data files found in {0}".format(itsdir)) + + # get list of iters from datafile names + print("") + print(" - Scanning \"{0}\"".format(itsdir)) + iters = [] + iter_pattern = "{0}{1}*([0-9]+)\.xml".format(itsdir, os.sep) + iter_re = re.compile(iter_pattern) + for df in datafiles: + match = iter_re.match(df) + if not match: + #error("BUG: Unknown line \nglob line = %s \npattern = %s" % (df, iter_pattern)) + continue + iters.append(int(match.group(1))) + iters.sort() + print(" - Found iters : {0}".format(str(iters)[1:-1])) + + # Initialise database + print(" - Initialising database file : \"{0}\"".format(outfile)) + if file_exist(outfile): + print(" ... File already exist. Deleting.") + try: + os.remove(outfile) + except: + error("Could not delete file {0}".format(outfile)) + db = sqlite3.connect(outfile, isolation_level=None) # manage our own transactions + + # some performance tuning + db.execute("PRAGMA temp_store = MEMORY;") + db.execute("PRAGMA synchronous = OFF;") + db.execute("PRAGMA journal_mode = OFF;") + + # parse xmml file to determine required database structure + print(" - Analysing model") + agents = {} + models = [model] + model_count = 0 + while len(models) > 0: + fname = models.pop() + model_count = model_count + 1 + if model_count == 1: + dirname = os.path.dirname(fname) + + # load xml file + print(" + parsing {0}".format(fname)) + try: + dom = minidom.parse(fname) + except IOError: + error("Unable to read model file ({0})".format(fname)) + except ExpatError: + error("Invalid XML file ({0})".format(fname)) + + # detect nested models + nodes = dom.getElementsByTagName("model") + for node in nodes: + status = get_node_text(node.getElementsByTagName("enabled")[0]) + if status != "true": continue # disabled. ignore + + # add nested model file to list of files + modelfile = get_node_text(node.getElementsByTagName("file")[0]) + models.append(os.path.join(dirname, modelfile)) + del(nodes) + + # detect agents + nodes = dom.getElementsByTagName("xagent") + for node in nodes: + # get agent name + aname = get_node_text(node.getElementsByTagName("name")[0]) + varlist = [] + vars = node.getElementsByTagName("variable") + # get list of agent memory vars + for var in vars: + varlist.append(get_node_text(var.getElementsByTagName("name")[0])) + del(vars) + + # remove array brackets + for i in range(len(varlist)): + v = varlist[i] + if "[" in v: varlist[i] = v[:v.index("[")] + + # add agent data to out list + if aname in agents.keys(): # agent already exist. Merge + for v in varlist: + if v not in agents[aname]: agents[aname].append(v) + else: # new agent + agents[aname] = varlist + del(nodes) + + # finished parsing this file + del(dom) + + # Create db tables + print(" - Initialising database structure") + print(" + creating table : _iters_") + db.execute("BEGIN;") + db.execute("CREATE TABLE _iters_ (itno)") # metadata table + + for a in agents: # agent data tables + #sql = "CREATE TABLE %s (_ITERATION_NO,%s)" % (a, ",".join(agents[a])) + sql = "CREATE TABLE {0} (_ITERATION_NO,{1})".format(a, ",".join(agents.get(a))) + print(" + creating table : {0}".format(a)) + db.execute(sql) + db.execute("END;") + + # Finally, load data from *.xml files + print(" - Importing snapshot data into database") + for i in iters: + db.execute("BEGIN;") + load_iteration_file(i, itsdir, db) + db.execute("END;") + + # Done! + print(" - DONE. Output file : {0}".format(outfile)) + print("") + + # close db + db.commit() + db.close() diff --git a/src/data_conversion_scripts/xml_db/gendb_special.py b/src/data_conversion_scripts/xml_db/gendb_special.py new file mode 100644 index 0000000..18879dd --- /dev/null +++ b/src/data_conversion_scripts/xml_db/gendb_special.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python +# $Id: gendb.py 2005 2009-07-31 14:14:48Z lsc $ +# +# Copyright (c) 2009 STFC Rutherford Appleton Laboratory +# Author: Lee-Shawn Chin +# Date : July 2009 +# +# NOTE: This is a proof-of-concept implementation. It was not designed for +# production use but merely to explore what is possible +# Use at your own risk ;) +# + +import argparse, os, re, sys +from xml.dom import minidom +from xml.parsers.expat import ExpatError +from glob import glob +import sqlite3 + +outfile = "iters.db" + +def error(mesg): + print(">>>>> (Error): {0}".format(mesg), file=sys.stderr) + sys.exit(1) + +def warn(mesg): + print(">> (Warning): {0}".format(mesg), file=sys.stderr) + +def file_exist(fname): + f = glob(fname) + if fname in f: return True + else: return False + +def get_node_text(node): + if len(node.childNodes) != 1 or node.childNodes[0].nodeType != node.TEXT_NODE: + error("getText() routine used on non-text XML node") + return node.childNodes[0].data.strip() + +def get_agent_sql(data): + # "data" should be in format + # data["name"] = "agent_name" + # data["iter"] = 1 + # data["vars"] = {} + # data["vars"]["varname"] = value + global agents + + try: + name = data["name"] + vars = data["vars"] + iter = data["iter"] + except: + error("BUG!! contact script author") + + # check agent exists + if name not in agents.keys(): + print("var ignored {0}".format(name)) + + else: + # retrieve values in the right order + keys = [] + values = [] + for v in vars.keys(): + if v == "name": continue + if v not in agents[name]: + continue + + else : + keys.append(v) + values.append(vars[v]) + + sql = [] + sql.append("insert into {0}".format(name)) + sql.append("(_ITERATION_NO,{0})".format(",".join(keys))) + sql.append("values ({0},'{1}');" .format(iter, "','".join(values))) + return " ".join(sql) + +def load_iteration_file(itno, basedir, db): + + filename = os.path.join(basedir, "{0}.xml".format(itno)) + print(" + Reading file {0}".format(filename)) + + if not file_exist(filename): + error("Strange! Input file disappeared : {0}".format(filename)) + try: + file = open(filename, "r") + except: + error("Could not open file {0}".format(filename)) + + + db.execute("INSERT INTO _iters_(itno) VALUES ({0});".format(i)) + + data = {} + in_xagent_tag = False + line = file.readline() + re_tag = re.compile(r"<(\w+)>([^<]*)") + + while line: + line = line.strip() + + if not line: # ignore empty lines + # get next line and restart loop + line = file.readline() + continue + + if in_xagent_tag: + if line == "": # end if this agent def + in_xagent_tag = False + + try: + db.execute(get_agent_sql(data)) # store data in db + except ValueError: + print("Skip agent") + del(data) + + else: # next agent mem + match = re_tag.match(line) + if not match: error("Unknown line: {0}".format(line)) + + tag = match.group(1) + val = match.group(2) + + if tag == "name": + data["name"] = val + else: + data["vars"][tag] = val + + else: # outside xagent tag + if line == "": + in_xagent_tag = True + data = {} + data["iter"] = i + data["vars"] = {} + + + line = file.readline() # get next line + + file.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(prog='gendb_special.py', description='Script creates SQLite db file from corresponding XML files in many-to-one fashion, and skips missing Agents.') + parser.add_argument('--model', '-m', help='Set (absolute or relative) path to the directory that contains the model, including the name of the model file (model.xml).', nargs=1, type=str, required=True) + parser.add_argument('--xml', '-x', help='Set (absolute or relative) path to the directory that contains the data to be converted (xml files).', nargs=1, type=str, required=True) + parser.add_argument('--output', '-o', help='Set (absolute or relative) path to the directory where the output file should be written (iters.db). If the folder does not exist, it will be created.', nargs=1, type=str, required=False) + args = parser.parse_args() + + # check if output argument is used + if args.output: + # check if a custom .db file is used + if args.output[0].rsplit('/', 1)[1].endswith('.db'): + # check if output directory exists + if not os.path.isdir(args.output[0].rsplit('/', 1)[0]): + error("Output directory ({0}) does not exist".format(args.output[0].rsplit('/', 1)[0])) + else: + outfile = str(args.output[0]) + else: + # check if output directory exists + if not os.path.isdir(args.output[0]): + error("Output directory ({0}) does not exist".format(args.output[0])) + else: + # check for output path and add an ending slash if necessary + if args.output[0][-1:] is '/': + outfile = str(args.output[0] + outfile) + else: + outfile = str(args.output[0] + '/' + outfile) + + # get input arguments + model = args.model[0] + itsdir = args.xml[0] + + # check that model file exists + if not file_exist(model): + error("Model file ({0}) does not exist".format(model)) + + # ensure directory exists + if not os.path.isdir(itsdir): + error("Input directory ({0}) does not exist".format(itsdir)) + + # Retrieve list of data files + datafiles = glob(os.path.join(itsdir, "[0-9]*.xml")) + if len(datafiles) == 0: + error("No data files found in {0}".format(itsdir)) + + # get list of iters from datafile names + print("") + print(" - Scanning \"{0}\"".format(itsdir)) + iters = [] + iter_pattern = "{0}{1}*([0-9]+)\.xml".format(itsdir, os.sep) + iter_re = re.compile(iter_pattern) + for df in datafiles: + match = iter_re.match(df) + if not match: + error("BUG: Unknown line \nglob line = {0} \npattern = {1}".format(df, iter_pattern)) + iters.append(int(match.group(1))) + iters.sort() + print(" - Found iters : {0}".format(str(iters)[1:-1])) + + # Initialise database + print(" - Initialising database file : \"{0}\"".format(outfile)) + if file_exist(outfile): + print(" ... File already exist. Deleting.") + try: + os.remove(outfile) + except: + error("Could not delete file {0}".format(outfile)) + db = sqlite3.connect(outfile, isolation_level=None) # manage our own transactions + + # some performance tuning + db.execute("PRAGMA temp_store = MEMORY;") + db.execute("PRAGMA synchronous = OFF;") + db.execute("PRAGMA journal_mode = OFF;") + + # parse xmml file to determine required database structure + print(" - Analysing model") + agents = {} + models = [model] + model_count = 0 + + while len(models) > 0: + fname = models.pop() + model_count = model_count + 1 + if model_count == 1: + dirname = os.path.dirname(fname) #update root model.xml folder only once, re-use for nested model.xml files with hard paths + + # load xml file + print(" + parsing {0}".format(fname)) + try: + dom = minidom.parse(fname) + except IOError: + error("Unable to read model file ({0})".format(fname)) + except ExpatError: + error("Invalid XML file ({0})".format(fname)) + + # detect nested models + nodes = dom.getElementsByTagName("model") + for node in nodes: + status = get_node_text(node.getElementsByTagName("enabled")[0]) + if status != "true": continue # disabled. ignore + + # add nested model file to list of files + modelfile = get_node_text(node.getElementsByTagName("file")[0]) + models.append(os.path.join(dirname,modelfile)) + del(nodes) + + # detect agents + nodes = dom.getElementsByTagName("xagent") + for node in nodes: + # get agent name + aname = get_node_text(node.getElementsByTagName("name")[0]) + varlist = [] + vars = node.getElementsByTagName("variable") + # get list of agent memory vars + for var in vars: + varlist.append(get_node_text(var.getElementsByTagName("name")[0])) + del(vars) + + # remove array brackets + for i in range(len(varlist)): + v = varlist[i] + if "[" in v: varlist[i] = v[:v.index("[")] + + # add agent data to out list + if aname in agents.keys(): # agent already exist. Merge + for v in varlist: + if v not in agents[aname]: agents[aname].append(v) + else: # new agent + agents[aname] = varlist + del(nodes) + + # finished parsing this file + del(dom) + + # Create db tables + print(" - Initialising database structure") + print(" + creating table : _iters_") + db.execute("BEGIN;") + db.execute("CREATE TABLE _iters_ (itno)") # metadata table + for a in agents: # agent data tables + sql = "CREATE TABLE {0} (_ITERATION_NO,{1})".format(a, ",".join(agents.get(a))) + print(" + creating table : {0}".format(a)) + db.execute(sql) + db.execute("END;") + + # Finally, load data from *.xml files + print(" - Importing snapshot data into database") + for i in iters: + db.execute("BEGIN;") + load_iteration_file(i, itsdir, db) + db.execute("END;") + + # Done! + print(" - DONE. Output file : {0}".format(outfile)) + print("") + + # close db + db.commit() + db.close() diff --git a/src/data_conversion_scripts/src/xml_hdf5/xml_hdf5.py b/src/data_conversion_scripts/xml_hdf5/xml_hdf5.py similarity index 100% rename from src/data_conversion_scripts/src/xml_hdf5/xml_hdf5.py rename to src/data_conversion_scripts/xml_hdf5/xml_hdf5.py diff --git a/src/data_conversion_scripts/misc/replace_xml_functions/replace_xml_functions.py b/src/replace_xml_writeout_functions/replace_xml_writeout_functions.py similarity index 100% rename from src/data_conversion_scripts/misc/replace_xml_functions/replace_xml_functions.py rename to src/replace_xml_writeout_functions/replace_xml_writeout_functions.py diff --git a/src/visualisation_scripts/config/config.yaml b/src/visualisation_scripts/config/config.yaml new file mode 100644 index 0000000..d0165ad --- /dev/null +++ b/src/visualisation_scripts/config/config.yaml @@ -0,0 +1,201 @@ +i/o: + # set up the name of the root folder from repository + repo_name: 'FLAViz' + + # set up your input_path for the resulting plots, it's relative unless it starts with a '/' + input_path: 'data/visualisation' + + #from main folder + #input_path: '../../data/visualisation' + + #from config folder: + #input_path: '../../../data/visualisation' + + #hard-coded: + #input_path: '/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/data/visualisation' + + # set up your output_path for the resulting plots, it's relative unless it starts with a '/' + output_path: 'results' + + #from main folder + #output_path: '../../results_test' + + #from config folder: + #output_path: '../../../results_test2' + + #hard-coded: + #output_path: '/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/results_test3' + + input_files: + CentralBank: CentralBank.h5 # please name the key as the agent name + Eurostat: Eurostat.h5 + Firm: Firm.h5 + +plot1: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [13] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: median + +plot2: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + var2: [total_debt] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 + +plot3: + timeseries: + agent: Firm + analysis: multiple_batch + variables: + var1: [price,total_debt,technology] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_percentile : 0.20 + upper_percentile : 0.80 + +plot4: + timeseries: + agent: Firm + analysis: agent + variables: + var1: [price] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,20]] + summary: mean + +plot5: + boxplot: + agent: Firm + analysis: multiple_set + variables: + var1: [price] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: full + +plot6: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + var2: [output] + delay: no + set: [13] + run: [range,[1,2]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot61: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + var2: [output] + delay: no + set: [13] + run: [range,[1,2]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot7: + scatterplot: + agent: Firm + analysis: agent + variables: + var1: [price] + delay: yes + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: full + +plot8: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + delay: yes + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot9: + histogram: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot10: + histogram: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean # Note: the summary is mean value, over each level, gives hist of distribution of means + +plot11: + histogram: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: full # Note: here the histogram is over the entire distribution + +plot12: + histogram: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10, 13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean diff --git a/TODO/visualisation/src/config_transform.yaml b/src/visualisation_scripts/config/config_transform.yaml similarity index 100% rename from TODO/visualisation/src/config_transform.yaml rename to src/visualisation_scripts/config/config_transform.yaml diff --git a/src/visualisation_scripts/config/plot_config.yaml b/src/visualisation_scripts/config/plot_config.yaml new file mode 100644 index 0000000..56719c9 --- /dev/null +++ b/src/visualisation_scripts/config/plot_config.yaml @@ -0,0 +1,152 @@ +plot1: + number_plots: one + plot_name: p1_one_set_multiple_runs_timeseries + plot_legend: yes + legend_location: best + ymin: 2.0 + ymax: 2.1 + xmin: 100 + xmax: 200 + xlabel: Time + ylabel: price + linestyle: solid + marker: None + +plot2: + number_plots: many + plot_name: p2_one_set_multiple_runs_ts_quantile + plot_legend: yes + legend_location: best + xlabel: Time + ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: green + plot_format: pdf + +plot3: + number_plots: one + plot_name: p3_ts_multibatch_analysis + plot_legend: yes + legend_location: best + xlabel: Time + ylabel: price + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot4: + number_plots: one + plot_name: p4_one_set_one_run_agentanalysis_timeseries + plot_legend: no + legend_location: best + xlabel: Time + ylabel: price + linestyle: solid + marker: None + greyscale: True + +plot5: + number_plots: one + plot_name: p5_one_set_one_run_bp_price + plot_legend: yes + legend_label: (Agent = Firm, var = Price) + legend_location: best + xlabel: Time + ylabel: Distribution over price + number_bars: 5 + greyscale: False + +plot6: + number_plots: one + plot_name: p6_one_set_multiple_runs_sp_price_output + plot_legend: yes + legend_location: best + legend_label: price vs. output + linestyle: solid + marker: + + +plot61: + number_plots: one + plot_name: p61_one_set_multiple_runs_sp_price_output + plot_legend: yes + ymin: 2.0 + ymax: 2.1 + xmin: 31 + xmax: 34 + legend_location: best + legend_label: price vs. output + linestyle: solid + marker: + + +plot7: + number_plots: one + plot_name: p7_one_set_agent_sp_price_delay + plot_legend: no + legend_location: best + legend_label: price delay vs. price + marker: + + +plot8: + number_plots: one + plot_name: p8_one_set_multiple_runs_sp_price_delay + plot_legend: yes + legend_location: best + legend_label: price delay vs. price + linestyle: solid + marker: o + +plot9: + number_plots: one + plot_name: p9_one_set_one_run_hg_price + plot_title: (Agent = Firm, var = Price) + bins: 50 + normed: 0 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + xlabel: xlabel + ylabel: ylabel + +plot10: + number_plots: one + plot_name: p10_one_set_multiple_runs_hg_price + plot_title: (Agent = Firm, var = Price) + bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + xlabel: xlabel + ylabel: ylabel + +plot11: + number_plots: one + plot_name: p11_one_set_multiple_runs_hg_price + plot_title: (Agent = Firm, var = Price) + bins: 50 + histtype: bar + plot_legend: yes + fill_between: yes + stacked: False + legend_location: best + xlabel: xlabel + ylabel: ylabel + +plot12: + number_plots: one + plot_name: p12_two_sets_multiple_runs_hg_price + plot_title: (Agent = Firm, var = Price) + bins: 50 + histtype: step + plot_legend: yes + fill_between: no + stacked: False + legend_location: best + xlabel: xlabel + ylabel: ylabel diff --git a/src/visualisation_scripts/config_full_examples/config.yaml b/src/visualisation_scripts/config_full_examples/config.yaml new file mode 100644 index 0000000..42a04e3 --- /dev/null +++ b/src/visualisation_scripts/config_full_examples/config.yaml @@ -0,0 +1,364 @@ +i/o: + # set up the name of the root folder from repository + repo_name: 'FLAViz' + #repo_name: 'ETACE' + + # set up your input_path for the resulting plots, it's relative to repo_name unless it starts with a '/' + input_path: 'data/visualisation' + + #from main folder + #input_path: '../../data/visualisation' + + #from config folder: + #input_path: '../../../data/visualisation' + + #hard-coded: + #input_path: '/media/sander/DataStorage1/GIT/GitHub/FLAViz@svdhoog/FLAViz/data/visualisation' + + # set up your output_path for the resulting plots, it's relative to repo_name unless it starts with a '/' + #output_path: 'results' + + #from main folder + output_path: 'results_test' + + input_files: + CentralBank: CentralBank.h5 # please name the key as the agent name + Eurostat: Eurostat.h5 + Firm: Firm.h5 + +# Multi-run plot: means across agents +plot0: + timeseries: + agent: Eurostat + analysis: multiple_run + variables: + var1: [monthly_output] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [0] + summary: no + +# Multi-run plot: does not work? +plot01: + timeseries: + agent: Eurostat + analysis: multiple_run + variables: + var1: [monthly_output] + set: [13] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [0] + summary: no + +# Multi-run plot: does not work? +plot02: + timeseries: + agent: Eurostat + analysis: multiple_run + variables: + var1: [monthly_output] + set: [10,13,16,17] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [0] + summary: no + + +# Multi-run plot: means across agents +plot1: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +# Multi-run plot: does not work? +plot11: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [13] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: no + +# Multi-run plot: does not work? +plot12: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10,13,16,17] + run: [range,[1,10]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: no + + +# Multi-run quantiles: +plot20: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + var2: [total_debt] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +# Multi-run quantiles: +plot21: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +# Multi-run quantiles: Plot quantiles for each run, across agent dimension: +plot22: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +# Multi-run quantiles: Plot quantiles for runs, across the agent dimension: +plot23: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + + +# Multi-run quantiles, many plots: Plots quantiles for each run, one plot per run +plot24: + timeseries: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +# Multi-batch quantiles: Plot quantiles for each set, across run dimension: +plot3: + timeseries: + agent: Firm + analysis: multiple_batch + variables: + var1: [price,total_debt,technology] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +# Multi-batch quantiles: Plot quantiles for each set, across run dimension: +plot30: + timeseries: + agent: Firm + analysis: multiple_batch + variables: + var1: [price,total_debt,technology] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +# Multi-set quantiles: plots quantiles over the full extent of multiple sets +plot31: + timeseries: + agent: Firm + analysis: multiple_set + variables: + var1: [price,total_debt,technology] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +plot4: + timeseries: + agent: Firm + analysis: agent + variables: + var1: [price] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,20]] + summary: mean + +plot41: + timeseries: + agent: Firm + analysis: agent + variables: + var1: [price] + set: [10,13,16,17] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: custom_quantile + quantile_values: + lower_quantile : 0.20 + upper_quantile : 0.80 + +plot5: + boxplot: + agent: Firm + analysis: multiple_set + variables: + var1: [price] + set: [13] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: full + +plot6: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + var2: [output] + delay: no + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot7: + scatterplot: + agent: Firm + analysis: agent + variables: + var1: [price] + delay: yes + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: full + +plot8: + scatterplot: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + delay: yes + set: [13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot9: + histogram: + agent: Firm + analysis: multiple_run + variables: + var1: [price] + set: [10] + run: [1] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean + +plot10: + histogram: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean # Note: the summary is mean value, over each level, gives hist of distribution of means + +plot11: + histogram: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: full # Note: here the histogram is over the entire distribution + +plot12: + histogram: + agent: Firm + analysis: multiple_batch + variables: + var1: [price] + set: [10, 13] + run: [range,[1,20]] + major: [range,[6020,12500,20]] + minor: [range,[1,80]] + summary: mean diff --git a/src/visualisation_scripts/config_full_examples/config_transform.yaml b/src/visualisation_scripts/config_full_examples/config_transform.yaml new file mode 100644 index 0000000..8a324c8 --- /dev/null +++ b/src/visualisation_scripts/config_full_examples/config_transform.yaml @@ -0,0 +1,15 @@ + +plot2: + variables: + var1: total_credit + var2: equity + transform_function: q_o_q + aggregate: mean + new_variables: + var1: total_credit_q_o_q + var2: equity_q_o_q + write_file: yes + output_file_name: 'sents.h5' + hdf_groupname: 'total_credit_ratio' + + diff --git a/src/visualisation_scripts/config_full_examples/plot_config.yaml b/src/visualisation_scripts/config_full_examples/plot_config.yaml new file mode 100644 index 0000000..44eb2de --- /dev/null +++ b/src/visualisation_scripts/config_full_examples/plot_config.yaml @@ -0,0 +1,256 @@ +plot0: + number_plots: one + plot_name: p0_one_set_one_run_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + +plot01: + number_plots: one + plot_name: p01_one_set_multiple_runs_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + +plot02: + number_plots: one + plot_name: p02_multi_set_multiple_runs_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + +plot1: + number_plots: one + plot_name: p1_one_set_one_run_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + +plot11: + number_plots: one + plot_name: p11_one_set_multiple_runs_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + +plot12: + number_plots: one + plot_name: p12_multi_set_multiple_runs_timeseries.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + +plot20: + number_plots: one + plot_name: p2_one_set_one_runs_2_vars_ts_quantile.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: red + + +plot21: + number_plots: one + plot_name: p21_one_set_one_runs_1_var_ts_quantile.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: red + +plot22: + number_plots: one + plot_name: p22_one_set_multiple_runs_ts_quantile.png + plot_legend: no + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: red + +plot23: + number_plots: one + plot_name: p23_multi_set_multiple_runs_ts_quantile.png + plot_legend: no + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: red + + +plot3: + number_plots: one + plot_name: p3_ts_multibatch_analysis_one_plot.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot30: + number_plots: many + plot_name: p30_ts_multibatch_analysis_many_plots.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot31: + number_plots: one + plot_name: p31_ts_multiset_analysis.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: y(t) + linestyle: solid + marker: None + fill_between: yes + fillcolor: darkgreen + +plot4: + number_plots: one + plot_name: p4_ts_agent_analysis_one_set_one_run.png + plot_legend: yes + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + greyscale: False + +plot41: + number_plots: one + plot_name: p41_ts_agent_analysis_multi_set_multi_run.png + plot_legend: no + legend_location: best + xaxis_label: Time + yaxis_label: price + linestyle: solid + marker: None + greyscale: False + +plot5: + number_plots: one + plot_name: p5_one_set_one_run_bp_price.png + plot_legend: yes + legend_label: (Agent = Firm, var = Price) + legend_location: best + xaxis_label: Time + yaxis_label: Distribution over price + number_bars: 5 + greyscale: True + +plot6: + number_plots: one + plot_name: p6_one_set_multiple_runs_sp_price_output.png + plot_legend: yes + legend_location: best + legend_label: price vs. output + linestyle: solid + marker: + + +plot7: + number_plots: one + plot_name: p7_one_set_agent_sp_price_delay.png + plot_legend: no + legend_location: best + legend_label: price delay vs. price + marker: + + +plot8: + number_plots: one + plot_name: p8_one_set_multiple_runs_sp_price_delay.png + plot_legend: yes + legend_location: best + legend_label: price delay vs. price + linestyle: solid + marker: o + +plot9: + number_plots: one + plot_name: p9_one_set_one_run_hg_price.png + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill: yes + stacked: False + legend_location: best + xaxis_label: xlabel + yaxis_label: ylabel + +plot10: + number_plots: one + plot_name: p10_one_set_multiple_runs_hg_price.png + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill: yes + stacked: False + legend_location: best + xaxis_label: xlabel + yaxis_label: ylabel + +plot11: + number_plots: one + plot_name: p11_one_set_multiple_runs_hg_price.png + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: bar + plot_legend: yes + fill: yes + stacked: False + legend_location: best + xaxis_label: xlabel + yaxis_label: ylabel + +plot12: + number_plots: one + plot_name: p12_two_sets_multiple_runs_hg_price.png + plot_title: (Agent = Firm, var = Price) + number_bins: 50 + histtype: step + plot_legend: yes + fill: no + stacked: False + legend_location: best + xaxis_label: xlabel + yaxis_label: ylabel diff --git a/TODO/visualisation/src/main.py b/src/visualisation_scripts/main.py similarity index 52% rename from TODO/visualisation/src/main.py rename to src/visualisation_scripts/main.py index 1f64e48..79151dc 100644 --- a/TODO/visualisation/src/main.py +++ b/src/visualisation_scripts/main.py @@ -11,16 +11,22 @@ def erf(msg): # function to output the error message and exit - print(" >> Error: %s" % msg) + print(" >> Error: {0}".format(msg)) sys.exit() def dir_check(d): if os.path.exists(d): print("- Directory ["+os.path.basename(d)+ "] is used for output files") - + #test + if args.verbose: + print("os.path.basename(d)="+os.path.basename(d)) + else: os.makedirs(d) print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") + #test + if args.verbose: + print("os.path.basename(d)="+os.path.basename(d)) def process_hdf_keys(string_in): # function to extract set and run values from set_*_run_* string @@ -35,7 +41,7 @@ def find_between(s, first, last): string_out = find_between(tmp_string, "/set_", "_iters") # TODO: trailing slash might cause error, so before passing on check if trailing slash present, if yes remove return list(map(int, string_out.split(','))) - +# unnecessary and can be removed def process_string(string_in): def find_between(s, first, last): try: @@ -44,8 +50,6 @@ def find_between(s, first, last): return s[start:end] except ValueError: return "" - operator = string_in.partition("[")[0] - string_out = find_between(string_in, "[", "]") return list([operator, float(string_out)]) @@ -109,32 +113,101 @@ def plt_histogram(): plot_function = {'timeseries': plt_timeseries, 'boxplot': plt_boxplot, 'histogram': plt_histogram, 'scatterplot': plt_scatterplot, 'transform': var_transform} return plot_function[key]() +# function to create a progressbar in verbose mode +def progress_bar(name, iteration, total, barLength=20): + percent = int(round((iteration / total) * 100)) + nb_bar_fill = int(round((barLength * percent) / 100)) + bar_fill = '#' * nb_bar_fill + bar_empty = ' ' * (barLength - nb_bar_fill) + #newline = '\n' + sys.stdout.write(str("\r{0}".format(name)).ljust(46) + "[{0}] {1}%".format(str(bar_fill + bar_empty), percent)) + sys.stdout.flush() + if __name__ == "__main__": - parser = argparse.ArgumentParser(prog='main.py', description='Visualise and transform various time-series data.') - parser.add_argument('parameterpath', help='Path to folder containing the parameter (.yaml) files', nargs=1, type=str) + parser = argparse.ArgumentParser(prog='main.py', description='FLAViz: Visualization and data transformation of timeseries data of agent-based models.') + parser.add_argument('--configpath', '-p', help='Path to folder that contains the configuration (.yaml) files (config.yaml, plot_config.yaml)', nargs=1, type=str, required=True) + parser.add_argument('--verbose', '-v', help='Activate the verbose mode which contains tracking steps and progress', required=False, action='store_false') args = parser.parse_args() - - P = main_configuration(args.parameterpath[0]) # instantiate main_configuration class to process main yaml files - inpath = P.input_fpath() + P = main_configuration(args.configpath[0]) # instantiate main_configuration class to process main yaml files + inpath = P.input_fpath() infiles = P.input_files() + + #test + #if args.verbose: + # print("\nPrint (main.py):") + # print("\n inpath="+inpath) + # print("\n infiles=") + # print(infiles) + # print("\n") + #print("\n outpath=") + #print(outpath) + primary_parameters = P.get_parameters() agent_storelist = {} # all the agent HDF files are stored in this dict + index = 0 for key, value in infiles.items(): f_p = str(inpath) + "/" + str(value) - agent_storelist[key] = pd.io.pytables.HDFStore(f_p) + #agent_storelist[key] = pd.io.pytables.HDFStore(f_p) + agent_storelist[key] = pd.io.pytables.HDFStore(f_p, mode='r') # Is read-only mode needed here? + # print a progressbar if verbose mode is activated + if not args.verbose: + index+=1 + progress_bar("Step 1: Preparing data structure ", index, len(infiles.items())) + if not args.verbose: + sys.stdout.write("\n") agent_dframes = {} # All the main dataframes of different agenttypes are stored in this dict + index = 0 + + #test + #print('\nPrint (main.py): agent_storelist.items()') #prints contents summary of h5 files + #print(agent_storelist.items()) + for agentname, agentstore in agent_storelist.items(): + + #test + #print("\nPrint (main.py): agentname="+agentname) + #print(agentname) + d = pd.DataFrame() # Main dataframe to hold all the dataframes of each instance (one agenttype) df_list = [] + + #test + #print('\nPrint (main.py): agentstore.keys()') + #print(agentstore.keys()) + for key in agentstore.keys(): # go through sets and runs in the HDF file sets_runs = process_hdf_keys(key) # get set and run values from the names: set_1_run_1_iters etc. hardcoded for set_*_run_*_iters atm + + ##test + # print('\nPrint (main.py): sets_runs') + # print('key='+key) + # print('process_hdf_keys(key)=') + # print(process_hdf_keys(key)) + # print('sets_runs=') + # print(sets_runs) #prints set nos. and contents summary of hdf5 + # print('sets_runs[0]= '+str(sets_runs[0])) #prints set nos. + # print('sets_runs[1]= '+str(sets_runs[1])) #prints content summary of hdf5 + s = sets_runs[0] r = sets_runs[1] - pnl = agentstore.select(key) # open datapanel for particular set and run - df = pnl.to_frame() # convert panel to Dataframe + #pnl = agentstore.select(key) # open datapanel for particular set and run + df = agentstore.select(key) # open datapanel for particular set and run + + #test + #print('\nPrint (main.py): pnl') + #print(pnl) + #print(pnl.shape) + + #df = pnl.to_frame() # convert panel to Dataframe + + #test + #print('\nPrint (main.py): df') + #print(df) + #print(df.shape) + # Add two columns for set and run into the dataframe for two added level of indexing df['set'] = s df['run'] = r @@ -142,30 +215,68 @@ def plt_histogram(): df.set_index('set', append=True, inplace=True) df_list.append(df.reorder_levels(['set', 'run', 'major', 'minor'])) + #test + #print('\nPrint (main.py): df_list') + #print(df_list) + + d = pd.concat(df_list) # Add each dataframe from panel into a main dataframe containing all sets and runs del df_list agent_dframes[agentname] = d # this dict contains agent-type names as keys, and the corresponding dataframes as values agentstore.close() + + #test + #print('\nPrint (main.py): d') + #print(d) + #print(agent_dframes[agentname]) + + # print a progressbar if verbose mode is activated + if not args.verbose: + index += 1 + progress_bar("Step 2: Processing data file " , index, len(agent_storelist.items())) + if not args.verbose: + sys.stdout.write("\n") del agent_storelist + index = 0 for idx, param in primary_parameters.items(): # read filter conditions from yaml frames = [] # list to store filtered dfs according to vars var_dic = {} for i, j in param['variables'].items(): if len(j) > 1: var_filter_list = [] - for s in range(1, len(j)): - var_filter_list.append(process_string(j[s])) - var_dic[j[0]] = var_filter_list + #for s in range(1, len(j)): + for s in range(0, len(j)): + #var_filter_list.append(process_string(j[s])) + var_dic[j[s]] = None + #var_dic[j[0]] = var_filter_list else: var_dic[j[0]] = None var_list = list(var_dic.keys()) - d = agent_dframes[param['agent']] # comment: this can be replaced in line below to save memory, here now just for simplicity - filtered = d.iloc[(d.index.get_level_values('set').isin(param['set'])) & (d.index.get_level_values('run').isin(param['run'])) & (d.index.get_level_values('major').isin(param['major'])) & (d.index.get_level_values('minor').isin(param['minor']))][var_list].dropna().astype(float) # stage-I filtering, all input vars are sliced with desired set & run values - #print filtered + ##memory-heavy version + #d = agent_dframes[param['agent']] # comment: this can be replaced in line below to save memory, here now just for simplicity. See memory-saving version + + ## check if table columns contain the given variables from config file + #for i, entry in enumerate(var_list): + # if not (entry in list(d)): + # erf("Table has columns {0} and var{1}='{2}' does not match.".format(list(d), i+1, entry)) + + ## stage-I filtering, all input vars are sliced with desired set & run values + #filtered = d.iloc[(d.index.get_level_values('set').isin(param['set'])) & (d.index.get_level_values('run').isin(param['run'])) & (d.index.get_level_values('major').isin(param['major'])) & (d.index.get_level_values('minor').isin(param['minor']))][var_list].dropna().astype(float) + + ##memory-saving version + # check if table columns contain the given variables from config file + for i, entry in enumerate(var_list): + if not (entry in list(agent_dframes[param['agent']])): + erf("Table has columns {0} and var{1}='{2}' does not match.".format(list(agent_dframes[param['agent']]), i+1, entry)) + + # stage-I filtering, all input vars are sliced with desired set & run values + filtered = agent_dframes[param['agent']].iloc[(agent_dframes[param['agent']].index.get_level_values('set').isin(param['set'])) & (agent_dframes[param['agent']].index.get_level_values('run').isin(param['run'])) & (agent_dframes[param['agent']].index.get_level_values('major').isin(param['major'])) & (agent_dframes[param['agent']].index.get_level_values('minor').isin(param['minor']))][var_list].dropna().astype(float) + df_main = pd.DataFrame() + index1 = 0 for dkey, dval in var_dic.items(): df = filter_by_value(dkey, dval, filtered) # stage-II filtering for selecting variables according to their values if df_main.empty: @@ -173,12 +284,19 @@ def plt_histogram(): else: df_main = pd.concat([df_main, df], axis=1) del df - summary_and_plot(idx, P, df_main, args.parameterpath[0]) # plot index, parameter object, data, parameter_filepath + + # print a progressbar if verbose mode is activated + if not args.verbose: + index1 += 1 + progress_bar("Step 3: Filtering/Plotting data for: {0} ".format(idx), index1, len(var_dic.items())) + if not args.verbose: + sys.stdout.write("\n") + summary_and_plot(idx, P, df_main, args.configpath[0]) # plot index, parameter object, data, parameter_filepath var_dic.clear() # clear dict of mapping between plot var and operator (for next cycle) del var_list[:] # clear the list of variables for next cycle ################################################################################################################################### -# TODO: add support for multiple agenttypes within a single plot, new entry in yaml (replace agent with, agent1, agent2), and parse +# TODO: add support for multiple agent types within a single plot, new entry in yaml (replace agent with, agent1, agent2), and parse # TODO: currently the filtering is done in two steps, find a way to do it in a single step # TODO: main data reprocessed for different types of plot (separate the processing and plot from loop to process main data just once diff --git a/TODO/visualisation/src/parameters.py b/src/visualisation_scripts/parameters.py similarity index 70% rename from TODO/visualisation/src/parameters.py rename to src/visualisation_scripts/parameters.py index 8379a48..f924fde 100644 --- a/TODO/visualisation/src/parameters.py +++ b/src/visualisation_scripts/parameters.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import yaml import argparse - +from pathlib import Path class A: # Types of analysis @@ -25,8 +25,7 @@ class NP: class main_configuration(): def __init__(self, in_fpath): - - self.main_f = in_fpath + '/main.yaml' # the main configuration file + self.main_f = in_fpath + '/config.yaml' # the main configuration file self.parsed_values = self.parse_yaml(self.main_f) def erf(self, msg): # function to output the error message and exit @@ -40,7 +39,7 @@ def parse_yaml(self, fname): # Function to parse input parameters from the main self.erf("unable to read file: %s" % fname) with f as stream: try: - p = yaml.load(stream) + p = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: # error-check for incorrect yaml syntax if hasattr(exc, 'problem_mark'): mark = exc.problem_mark @@ -84,32 +83,49 @@ def get_parameters(self): req_par =['agent','analysis','variables','set','run','major','minor','summary'] for p in req_par: if p not in inner_dic[self.get_plotname_by_idx(key)].keys(): - self.erf(p + " not specified in configuration file") + self.erf(p + " not specified in configuration file") for v in inner_dic.values(): d[key] = self.process_parameters(v) return d def input_fpath(self): if 'i/o' in self.parsed_values.keys(): - return self.parsed_values['i/o']['input_path'] + path_config = self.parsed_values['i/o']['input_path'] + if path_config[:1] == '/': # check for full path inside config + input_path = str(Path(path_config)) + return input_path + else: # use the path from config as relative path + repo_name = self.parsed_values['i/o']['repo_name'] + repo_path = str(Path("").absolute()).rsplit(repo_name, 1)[0] + repo_name + input_path = repo_path + '/' + path_config # fix to repo root + return input_path return self.erf("Missing input path!") def output_fpath(self): if 'i/o' in self.parsed_values.keys(): - return self.parsed_values['i/o']['output_path'] + #output_path = Path(self.parsed_values['i/o']['output_path']) + path_config = self.parsed_values['i/o']['output_path'] + if path_config[:1] == '/': # check for full path inside config + output_path = str(Path(path_config)) + return output_path + else: # use the path from config as relative path + repo_name = self.parsed_values['i/o']['repo_name'] + repo_path = str(Path("").absolute()).rsplit(repo_name, 1)[0] + repo_name + output_path = repo_path + '/' + path_config + return output_path return self.erf("Missing output path!") def input_files(self): if 'i/o' in self.parsed_values.keys(): return self.parsed_values['i/o']['input_files'] - return self.erf("Missing input path!") + return self.erf("Missing input files!") class transform_configuration(): def __init__(self, in_fpath): - - self.transform_f = in_fpath + '/transform.yaml' # the main configuration file + + self.transform_f = in_fpath + '/config_transform.yaml' # the configuration file for variable transformations self.parsed_values = self.parse_yaml(self.transform_f) def erf(self, msg): # function to output the error message and exit @@ -123,7 +139,7 @@ def parse_yaml(self, fname): # Function to parse input parameters from the main self.erf("unable to read file: %s" % fname) with f as stream: try: - p = yaml.load(stream) + p = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: # error-check for incorrect yaml syntax if hasattr(exc, 'problem_mark'): mark = exc.problem_mark @@ -132,7 +148,7 @@ def parse_yaml(self, fname): # Function to parse input parameters from the main print(" >> Unknown problem with %s file:" % fname) sys.exit() return p - + def get_parameters(self, idx): inner_dic = self.parsed_values[idx] return inner_dic @@ -142,10 +158,8 @@ class Plot_configuration(): def __init__(self, p_fpath): - self.plot_f = p_fpath + '/plot.yaml' + self.plot_f = p_fpath + '/plot_config.yaml' # the configuration file for plot options (matplotlib) self.__param = self.parse_yaml(self.plot_f) - - self.__default_plot_parameters = Figure_default_parameters() def erf(self, msg): @@ -166,7 +180,7 @@ def parse_yaml(self, fname): self.erf("unable to read file: %s" % fname) with f as stream: try: - p = yaml.load(stream) + p = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: # error-check for incorrect yaml syntax if hasattr(exc, 'problem_mark'): mark = exc.problem_mark @@ -201,15 +215,15 @@ def num_plots(self, key): return self.__param[key]['number_plots'] return self.__default_plot_parameters.__dict__['number_plots'] - def y_label(self, key): - if 'yaxis_label' in self.__param[key].keys(): - return self.__param[key]['yaxis_label'] - return self.__default_plot_parameters.__dict__['yaxis_label'] + def ylabel(self, key): + if 'ylabel' in self.__param[key].keys(): + return self.__param[key]['ylabel'] + return self.__default_plot_parameters.__dict__['ylabel'] - def x_label(self, key): - if 'xaxis_label' in self.__param[key].keys(): - return self.__param[key]['xaxis_label'] - return self.__default_plot_parameters.__dict__['xaxis_label'] + def xlabel(self, key): + if 'xlabel' in self.__param[key].keys(): + return self.__param[key]['xlabel'] + return self.__default_plot_parameters.__dict__['xlabel'] def plot_title(self, key): if 'plot_title' in self.__param[key].keys(): @@ -221,15 +235,30 @@ def plot_name(self, key): return self.__param[key]['plot_name'] return self.__default_plot_parameters.__dict__['plot_name'] - def llim(self, key): - if 'l_lim' in self.__param[key].keys(): - return self.__param[key]['l_lim'] - return self.__default_plot_parameters.__dict__['l_lim'] + def plot_format(self, key): + if 'plot_format' in self.__param[key].keys(): + return self.__param[key]['plot_format'] + return self.__default_plot_parameters.__dict__['plot_format'] - def ulim(self, key): - if 'u_lim' in self.__param[key].keys(): - return self.__param[key]['u_lim'] - return self.__default_plot_parameters.__dict__['u_lim'] + def xmin(self, key): + if 'xmin' in self.__param[key].keys(): + return self.__param[key]['xmin'] + return self.__default_plot_parameters.__dict__['xmin'] + + def xmax(self, key): + if 'xmax' in self.__param[key].keys(): + return self.__param[key]['xmax'] + return self.__default_plot_parameters.__dict__['xmax'] + + def ymin(self, key): + if 'ymin' in self.__param[key].keys(): + return self.__param[key]['ymin'] + return self.__default_plot_parameters.__dict__['ymin'] + + def ymax(self, key): + if 'ymax' in self.__param[key].keys(): + return self.__param[key]['ymax'] + return self.__default_plot_parameters.__dict__['ymax'] def linestyle(self, key): if 'linestyle' in self.__param[key].keys(): @@ -257,9 +286,9 @@ def facecolors(self, key): return self.__default_plot_parameters.__dict__['facecolors'] def bins(self, key): - if 'number_bins' in self.__param[key].keys(): - return self.__param[key]['number_bins'] - return self.__default_plot_parameters.__dict__['number_bins'] + if 'bins' in self.__param[key].keys(): + return self.__param[key]['bins'] + return self.__default_plot_parameters.__dict__['bins'] def histtype(self, key): if 'histtype' in self.__param[key].keys(): @@ -271,25 +300,25 @@ def stacked(self, key): return self.__param[key]['stacked'] return self.__default_plot_parameters.__dict__['stacked'] - def norm(self, key): + def normed(self, key): if 'normed' in self.__param[key].keys(): return self.__param[key]['normed'] return self.__default_plot_parameters.__dict__['normed'] - def fill(self, key): - if 'fill' in self.__param[key].keys(): - return self.__param[key]['fill'] - return self.__default_plot_parameters.__dict__['fill'] + def fill_between(self, key): + if 'fill_between' in self.__param[key].keys(): + return self.__param[key]['fill_between'] + return self.__default_plot_parameters.__dict__['fill_between'] def fillcolor(self, key): if 'fillcolor' in self.__param[key].keys(): return self.__param[key]['fillcolor'] return self.__default_plot_parameters.__dict__['fillcolor'] - def numboxplotbars(self, key): - if 'number_bars' in self.__param[key].keys(): - return self.__param[key]['number_bars'] - return self.__default_plot_parameters.__dict__['number_bars'] + def greyscale(self, key): + if 'greyscale' in self.__param[key].keys(): + return self.__param[key]['greyscale'] + return self.__default_plot_parameters.__dict__['greyscale'] class Figure_default_parameters(object): @@ -298,27 +327,28 @@ def __init__(self): self.legend_label = None self.legend_location = 'best' self.plot_type = None - self.number_plots = 'one' - self.plot_name = 'default_fig.png' - self.l_lim = None - self.u_lim = None - self.linestyle = 1 + self.number_plots = 'many' + self.plot_name = None + self.plot_format = 'png' + self.xmin = None + self.xmax = None + self.ymin = None + self.ymax = None + self.linestyle = 'solid' self.marker = 4 self.markerfacecolor = None self.markersize = None - # self.label = None self.facecolors = None self.plot_title = None - self.xaxis_label = None - self.yaxis_label = None - self.number_bins = 50 + self.xlabel = None + self.ylabel = None + self.bins = 50 self.histtype = 'bar' self.stacked = False self.normed = 1 - self.fill = False + self.fill_between = False self.fillcolor = 'black' - self.number_bars = 5 - + self.greyscale = False if __name__ == "__main__": diff --git a/src/visualisation_scripts/plots.py b/src/visualisation_scripts/plots.py new file mode 100644 index 0000000..44b8e2f --- /dev/null +++ b/src/visualisation_scripts/plots.py @@ -0,0 +1,1937 @@ +from parameters import NP, A, Plot_configuration +from summarystats import SummaryStats +import sys, os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +#plt.style.use('ggplot') +import matplotlib.cm as cm + + + +class Plot(): + def __init__(self, idx, data, par_fpath): + self.idx = idx + self.__data = data + self.__P = Plot_configuration(par_fpath) + + def num_plot_mapper(self, val, obj): + one_plot = lambda: obj.one_plot() + many_plot = lambda: obj.many_output() + options = {'one': one_plot, 'many': many_plot} + return options[val]() + + def timeseries(self, main_param, outpath): + T = Timeseries(self.idx, self.__data, self.__P, main_param, outpath) + self.num_plot_mapper(self.__P.num_plots(self.idx), T) + + def boxplot(self, main_param, outpath): + B = Boxplot(self.idx, self.__data, self.__P, main_param, outpath) + self.num_plot_mapper(self.__P.num_plots(self.idx), B) + + def scatterplot(self, main_param, outpath): + S = Scatterplot(self.idx, self.__data, self.__P, main_param, outpath) + self.num_plot_mapper(self.__P.num_plots(self.idx), S) + + def histogram(self, main_param, outpath): + H = Histogram(self.idx, self.__data, self.__P, main_param, outpath) + self.num_plot_mapper(self.__P.num_plots(self.idx), H) + + +class Timeseries(A): + + def __init__(self, idx, data, plt_config, main_param, outpath): + self.idx = idx + self.agent = main_param['agent'] + self.variables = [item for sublist in (list(main_param['variables'].values())) for item in sublist] + self.__data = data + self.__N = len(main_param['major']) #length of major axis: iterations + self.__S = len(main_param['set']) #length of sets list + self.__R = len(main_param['run']) #length of runs list + self.__analysistype = self.map_analysis(main_param['analysis']) + self.analysistype = main_param['analysis'] + self.__P = plt_config + self.summary = main_param['summary'] + self.outpath = outpath + '/timeseries' + self.dir_check(self.outpath) + + def map_analysis(self, val): + analysis_values = {'agent': A.agent, 'multiple_run': A.multiple_run, 'multiple_batch': A.multiple_batch, 'multiple_set': A.multiple_set} + return analysis_values[val] + + # Function to check for existing directories, and create a new one if not present + def dir_check(self, d): + if os.path.exists(d): + print("- Directory ["+os.path.basename(d)+ "] is used for output files") + else: + os.makedirs(d) + print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") + + def plot_line(self, ax, x, y, l_label, clr, forced_ylabel): + if self.__P.legend_label(self.idx) is None: + le_label = l_label + else: + le_label = self.__P.legend_label(self.idx) + out = ax.plot(x, y, linestyle=self.__P.linestyle(self.idx), marker=self.__P.marker(self.idx), + markerfacecolor=self.__P.markerfacecolor(self.idx), markersize=self.__P.markersize(self.idx), label=le_label, color = clr) + + #Axes labels + if self.__P.xlabel(self.idx) is None or self.__P.xlabel(self.idx) == 'Auto' or self.__P.xlabel(self.idx) == 'auto': + ax.set_xlabel("Time") + else: + ax.set_xlabel(self.__P.xlabel(self.idx)) + + if self.__P.ylabel(self.idx) is None or self.__P.ylabel(self.idx) == 'Auto' or self.__P.ylabel(self.idx) == 'auto': + ax.set_ylabel(str(self.agent) + ' : ' + forced_ylabel) + else: + ax.set_ylabel(self.__P.ylabel(self.idx)) + + #No axes labels + if self.__P.xlabel(self.idx) == 'None' or self.__P.xlabel(self.idx) == 'none': + ax.set_xlabel("") + + if self.__P.ylabel(self.idx) == 'None' or self.__P.ylabel(self.idx) == 'none': + ax.set_ylabel("") + + plt.axis([self.__P.xmin(self.idx),self.__P.xmax(self.idx),self.__P.ymin(self.idx),self.__P.ymax(self.idx)]) + + if self.__P.legend(self.idx) is True: + ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) + + return out + + + def one_plot(self): + file_count = 0 + step = 1 + if self.summary == 'custom_quantile': + step = 2 + + for col in range(0, len(self.__data.columns), step): + if self.summary == 'custom_quantile': + dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() # one variable, one case at a time + else: + dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) + + legend_label = dframe.columns + fig, ax = plt.subplots() + + # timeseries one_plot agent + if self.__analysistype == A.agent: + minor_index = dframe.index.get_level_values('minor').unique() + for i, m in enumerate(minor_index): + D = dframe.xs(int(m), level='minor') + if len(D.columns) == 2: + print("Quantile not possible for agent level analysis!") + sys.exit(1) + else: + y = [] + for l in range(0, len(D), self.__N): + y.append(np.array(D[l:l+self.__N])) + x = np.arange(1, self.__N+1) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(D)//self.__N))) + size = len(D)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = i/len(minor_index) + #print("First a: ", a) + #print(np.random.uniform(0, 1, size = len(D)//self.__N)) + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + # Index values: + print("D.index.levels:") + #print(D.index.levels) #D only contains sets, runs, iters, no agents + + print("sets:") + print('D.index.levels[0].values') + print(D.index.levels[0].values) #set values + + print("runs:") + print('D.index.levels[1].values') + print(D.index.levels[1].values) #run values + # print("iters:") + # print('D.index.levels[2].values') + # print(D.index.levels[2].values) #iter values + + nsets = len(D.index.levels[0]) + nruns = len(D.index.levels[1]) + + print("Length set list:") + print(nsets) + + print("Length run list:") + print(nruns) + + for r in range(0, len(y)): + clr = next(colors) + + # Index values: block in r-loop + run_idx = r % nruns # r MOD blocks, ex: r MOD 4 + set_idx = r // nruns # r DIV blocks, ex: r DIV 4 + + #TEST + # print("set:") + # print(D.index.levels[0].values[set_idx]) + + # print("run:") + # print(D.index.levels[1].values[run_idx]) + + ##print("iters:") + ##print(D.index.levels[2].values) + + #Agent index: does not exist in D, but does in dframe + #TODO: use dframe or d or df_main (all not defined n this function) + # print("agent index:") + # print(m) + + #Use run and set labels derived from row index r + set_no = D.index.levels[0].values[set_idx] + run_no = D.index.levels[1].values[run_idx] + + #TODO: Use agent label derived from agent index m (required access to global dataframe) + agent_ind = m + #agent_id = df_main.index.levels[3].values[m] #TODO + + self.plot_line(ax, x, y[r], 'set ' + str(set_no) + ' run ' + str(run_no) + ' agent '+ str(agent_ind), clr, str(self.__data.columns[col])) # Legend entries for runs + + #Timeseries plot_name one_plot agent case 0 + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_VariableName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_VariableName + + #Test code + #if args.trace: + print("Timeseries [case 0 one_plot analysis=Agent]: ") #agent analysis, one_plot + print(plot_name) + #End of Test code + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + else: # timeseries one_plot quantiles multiple_run,multiple_batch,multiple_set + #Test + # print('len(dframe.columns)= '+str(len(dframe.columns))) + # print(dframe) + + if len(dframe.columns) == 2: # quantiles + + print('- Summary [quantiles]') + + y1 = [] + y2 = [] + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + + x = np.arange(1, self.__N+1) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + # Block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, summary across runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + #print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, summary across batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + for r in range(0, len(dframe)//self.__N): + clr = next(colors) + + # block set_no run_no + if(self.analysistype == 'multiple_run'): + run_idx = r % nruns + set_idx = r // nruns + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values[run_idx] + self.plot_line(ax, x, y1[r], legend_label[0]+' set ' + str(set_no)+' run ' + str(run_no), clr, str(self.__data.columns[col])) # Legend entries for quantiles per set + self.plot_line(ax, x, y2[r], legend_label[1]+' set ' + str(set_no)+' run ' + str(run_no), clr, str(self.__data.columns[col])) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("run:", dframe.index.levels[1].values[run_idx]) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + if(self.analysistype == 'multiple_batch'): + #run_idx = r % nruns #irrelevant? + set_idx = r + set_no = dframe.index.levels[0].values[set_idx] + run_no = 0 + self.plot_line(ax, x, y1[r], legend_label[0]+' set ' + str(set_no), clr, str(self.__data.columns[col])) # Legend entries for quantiles per set + self.plot_line(ax, x, y2[r], legend_label[1]+' set ' + str(set_no), clr, str(self.__data.columns[col])) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("iters:", dframe.index.levels[1].values) #run values + if(self.analysistype == 'multiple_set'): + set_no = self.summary + run_no = 0 + self.plot_line(ax, x, y1[r], legend_label[0]+' set ' + str(set_no), clr, str(self.__data.columns[col])) # Legend entries for quantiles per set + self.plot_line(ax, x, y2[r], legend_label[1]+' set ' + str(set_no), clr, str(self.__data.columns[col])) + #print("iters:", dframe.index.values) #iter values + + if self.__P.fill_between(self.idx): + plt.fill_between(x, y1[r], y2[r], color=self.__P.fillcolor(self.idx), alpha=.5) + + #Timeseries plot_name one_plot not-agent not-full case 1 + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_VariableName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_VariableName + + #Test code + #if args.trace: + print("Timeseries [case 1 one_plot, analysis != Agent, quantiles]: ") #multiple_batch + print(plot_name) + #End of Test code + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + else: # timeseries one_plot !quantiles multiple_run,multiple_batch,multiple_set + + print('- Summary [!quantiles?]') + + y1 = [] + col_A = dframe[dframe.columns[0]] + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + # Block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, summary across runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + #print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, summary across batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + for r in range(0, len(dframe)//self.__N): + x = np.arange(1, self.__N+1) + clr = next(colors) + + # Block inside r-loop: set_no run_no + if(self.analysistype == 'multiple_run'): + run_idx = r % nruns + set_idx = r // nruns + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values[run_idx] + self.plot_line(ax, x, y1[r], 'set ' + str(set_no) + ' ' + self.summary, clr, str(self.__data.columns[col])) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("run:", dframe.index.levels[1].values[run_idx]) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + if(self.analysistype == 'multiple_batch'): + #run_idx = r % nruns #irrelevant? + set_idx = r + set_no = dframe.index.levels[0].values[set_idx] + run_no = 0 + self.plot_line(ax, x, y1[r], 'set ' + str(set_no) + ' ' + self.summary, clr, str(self.__data.columns[col])) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("iters:", dframe.index.levels[1].values) #run values + if(self.analysistype == 'multiple_set'): + set_no = self.summary + run_no = 0 + self.plot_line(ax, x, y1[r], 'set ' + str(set_no) + ' ' + self.summary, clr, str(self.__data.columns[col])) + #print("iters:", dframe.index.values) #iter values + + #Timeseries plot_name one_plot not-agent full case 2 + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_VariableName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_VariableName + + #Test + #if args.trace: + print("Timeseries [case 2 one_plot, analysis != Agent, !quantiles]:") + print(plot_name) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + file_count = file_count + 1 + + def many_output(self): + step = 1 + if self.summary == 'custom_quantile': + step = 2 + file_count = 0 + for col in range(0, len(self.__data.columns),step): + print('- Plotting [' + self.__data.columns[col] + ']') + if self.summary == 'custom_quantile': + dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() # one variable, one case at a time + else: + dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) + legend_label = dframe.columns + + # timeseries many_plot agent + if self.__analysistype == A.agent: + print('- Warning: analysis type is Agent, plotting type is many_output.') + print('- Number of plots: ' + str(len(dframe)//self.__N) + ' of ' + str(len(self.__data.columns)*(len(dframe)//self.__N))) + + minor_index = dframe.index.get_level_values('minor').unique() # get the index values for minor axis, which will later be used to sort the dataframe + + #Use groupby + grouped = dframe.groupby('minor') + + for m in minor_index: + D = grouped.get_group(m) + #D = dframe.xs( int(m) , level='minor') # remove + if len(D.columns) == 2: + print("Quantile not possible for agent level analysis") + sys.exit(1) + else: + count = 0 + + # set plot colors for output files + size = len(D)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + nsets = len(D.index.levels[0]) + nruns = len(D.index.levels[1]) + + for i in range(0,len(D),self.__N): # blocks of self.__N rows belong to one agent + #fig, ax = plt.subplots() + y = np.array(D[i:i+self.__N]) + x = np.arange(1, self.__N+1) + #clr = next(colors) + + for ind, r in enumerate(range(0, len(D)//self.__N)): # loop all rows/no.iters + # print('Loop dframe block r: '+str(ind)+'/'+str(len(D)//self.__N)) + + fig, ax = plt.subplots() + clr = next(colors) + + # Index values: + run_idx = r % nruns # r MOD blocks, ex: r MOD 4 + set_idx = r // nruns # r DIV blocks, ex: r DIV 4 + + #Use run and set labels derived from row index r + set_no = D.index.levels[0].values[set_idx] + run_no = D.index.levels[1].values[run_idx] + + #TODO: Use agent label derived from agent index m (required access to global dataframe) + agent_ind = m + #agent_id = df_main.index.levels[3].values[m] #TODO + + self.plot_line(ax, x, y, 'set ' + str(set_no) + ' run ' + str(run_no) + ' agent ' + str(m), clr, str(self.__data.columns[col])) + + #Timeseries plot_name many_output agent case 3 + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) + '_set_' + str(set_no) + '_run_' + str(run_no) + "_agent_" + str(m)#PlotLabel_AgentName_VariableName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Add a counter to the end + # if count != 0: + # plot_name = str(plot_name) + '_' + str(count) + + #Test + #if args.trace: + print("Timeseries [case 3 many_output analysis=Agent]:") + print(plot_name) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + count = count + 1 + else: # timeseries many_plot quantiles multiple_run,multiple_batch,multiple_set + if len(dframe.columns) == 2: # quantiles + y1 = [] + y2 = [] + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + x = np.arange(1, self.__N+1) + + # set plot colors for output files + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + # block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, summary across runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + #print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, summary across batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + # many plot + for r in range(0, len(dframe)//self.__N): + fig, ax = plt.subplots() + clr = next(colors) + + if(self.analysistype == 'multiple_run'): + run_idx = r % nruns + set_idx = r // nruns + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values[run_idx] + self.plot_line(ax, x, y1[r], 'set ' + str(set_no) + ' run ' + str(run_no), clr, str(self.__data.columns[col])) # Legend entries for quantiles per set + self.plot_line(ax, x, y2[r], 'set ' + str(set_no) + ' run ' + str(run_no), clr, str(self.__data.columns[col])) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("run:", dframe.index.levels[1].values[run_idx]) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + if(self.analysistype == 'multiple_batch'): + #run_idx = r % nruns #Irrelevant? + set_idx = r % nsets + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values #names + self.plot_line(ax, x, y1[r], 'set ' + str(set_no), clr, str(self.__data.columns[col])) # Legend entries for quantiles per set + self.plot_line(ax, x, y2[r], 'set ' + str(set_no), clr, str(self.__data.columns[col])) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("iters:", dframe.index.levels[1].values) #run values + if(self.analysistype == 'multiple_set'): + #run_idx = r % nruns #Irrelevant? + #set_idx = r // nruns + set_no = self.summary + run_no = 0 + self.plot_line(ax, x, y1[r], 'set ' + str(set_no), clr, str(self.__data.columns[col])) # Legend entries for quantiles per set + self.plot_line(ax, x, y2[r], 'set ' + str(set_no), clr, str(self.__data.columns[col])) + #print("iters:", dframe.index.values) #iter values + + if self.__P.fill_between(self.idx): + plt.fill_between(x, y1[r], y2[r], color='k', alpha=.5) + + #Timeseries plot_name many_output not-agent not-full case 4 + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) + '_set_' + str(set_no) + '_run_' + str(run_no) #PlotLabel_AgentName_VariableName + + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + + #Add a counter to the end + # if file_count != 0: + # plot_name = str(plot_name) + '_' + str(file_count) + + #Test + #if args.trace: + print("Timeseries [case 4 many_output, analysis != Agent, quantiles]:") + print(plot_name) + #print("file_count= "+str(file_count)) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + file_count = file_count + 1 + else: # timeseries many_plot !quantiles multiple_run,multiple_batch,multiple_set + y =[] + for i in range(0,len(dframe),self.__N): + y.append(np.array(dframe[i:i+self.__N])) + + # set plot colors for output files + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + # block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, summary across runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + #print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, summary across batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + # many plot + for r in range(0, len(dframe)//self.__N): + fig, ax = plt.subplots() + x = np.arange(1, self.__N+1) + clr = next(colors) + + if(self.analysistype == 'multiple_run'): + run_idx = r % nruns + set_idx = r // nruns + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values[run_idx] + self.plot_line(ax, x, y[r], 'set ' + str(set_no) + ' run ' + str(run_no), clr, str(self.__data.columns[col])) + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) + '_set_' + str(set_no) + ' run ' + str(run_no) #PlotLabel_AgentName_VariableName + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("run:", dframe.index.levels[1].values[run_idx]) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + if(self.analysistype == 'multiple_batch'): + #run_idx = r % nruns #Irrelevant? + set_idx = r % nsets + set_no = dframe.index.levels[0].values[set_idx] + run_no = "" #dframe.index.levels[1].values #names + self.plot_line(ax, x, y[r], 'set ' + str(set_no), clr, str(self.__data.columns[col])) + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) + '_set_' + str(set_no) #PlotLabel_AgentName_VariableName # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("iters:", dframe.index.levels[1].values) #run values + if(self.analysistype == 'multiple_set'): + #run_idx = r % nruns #Irrelevant? + #set_idx = r // nruns + set_no = self.summary + run_no = "" #0 + self.plot_line(ax, x, y[r], 'set ' + str(set_no), clr, str(self.__data.columns[col])) + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) + '_set_' + str(set_no) #PlotLabel_AgentName_VariableName + #print("iters:", dframe.index.values) #iter values + + #Timeseries plot_name many_output not-agent not-full case 5 + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_VariableName + + #Add a counter to the end + # if file_count != 0: + # plot_name = str(plot_name) + '_' + str(file_count) + + #Test + #if args.trace: + print("Timeseries [case 5 many_output, analysis != Agent, !quantiles]:") + print(plot_name) + #print("file_count= "+str(file_count)) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + +class Histogram(): + + def __init__(self, idx, data, plt_config, main_param, outpath): + self.idx = idx + self.agent = main_param['agent'] + self.__data = data + self.outpath = outpath + '/histogram' + self.dir_check(self.outpath) + #self.__N = len(main_param['major']) + self.__analysistype = self.map_analysis(main_param['analysis']) + self.analysistype = main_param['analysis'] + self.__P = plt_config + self.summary = main_param['summary'] + if self.__analysistype == A.agent and self.summary == 'custom_quantile': + print(">> Quantile not possible for agent level analysis!") + sys.exit(1) + + def map_analysis(self, val): + analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} + return analysis_values[val] + + # Function to check for existing directories, and create a new one if not present + def dir_check(self, d): + if os.path.exists(d): + print("- Directory ["+os.path.basename(d)+ "] is used for output files") + else: + os.makedirs(d) + print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") + + def plot_histogram(self, ax, data, label, colors, n_bins, forced_xlabel): + + if self.__P.legend_label(self.idx) is None: + le_label = label + else: + le_label = self.__P.legend_label(self.idx) + out = ax.hist(data, n_bins, histtype=self.__P.histtype(self.idx), stacked=self.__P.stacked(self.idx), density=self.__P.normed(self.idx), fill=self.__P.fill_between(self.idx), color=colors, alpha=.5, label=le_label) + + if self.__P.plot_title(self.idx) is not None: + ax.set_title(self.__P.plot_title(self.idx)) + + #Axes labels + if self.__P.xlabel(self.idx) is None or self.__P.xlabel(self.idx) == 'Auto' or self.__P.xlabel(self.idx) == 'auto': + ax.set_ylabel(str(self.agent) + ' : ' + forced_xlabel) + else: + ax.set_xlabel(self.__P.xlabel(self.idx)) + + if self.__P.ylabel(self.idx) is None or self.__P.ylabel(self.idx) == 'Auto' or self.__P.ylabel(self.idx) == 'auto': + ax.set_ylabel(str("count")) + else: + ax.set_ylabel(self.__P.ylabel(self.idx)) + + #No axes labels + if self.__P.xlabel(self.idx) == 'None' or self.__P.xlabel(self.idx) == 'none': + ax.set_xlabel("") + + if self.__P.ylabel(self.idx) == 'None' or self.__P.ylabel(self.idx) == 'none': + ax.set_ylabel("") + + plt.axis([self.__P.xmin(self.idx),self.__P.xmax(self.idx),self.__P.ymin(self.idx),self.__P.ymax(self.idx)]) + + if self.__P.legend(self.idx) is True: + ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) + + #TODO: legend for histogram + #if self.__P.legend(self.idx) is True: + # ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) + + return out + + + def one_plot(self): + file_count = 0 + step = 1 + if self.summary == 'custom_quantile': + step = 2 + + for col in range(0, len(self.__data.columns), step): # one variable, one case at a time + if self.summary == 'custom_quantile': + dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy().dropna() # hist method does not support NaN + else: + dframe = pd.DataFrame(self.__data[self.__data.columns[col]]).dropna() + + self.__N = len(dframe.index.get_level_values('major').unique()) + + legend_label = dframe.columns + if self.__analysistype == A.agent: + minor_index = dframe.index.get_level_values('minor').unique() + fig, ax = plt.subplots() + for i, m in enumerate(minor_index): + D = dframe.xs(int(m), level='minor') + if len(D.columns) == 2: # TODO: this check done in class constructor, so no need + print("Quantile not possible for agent level analysis!") + sys.exit(1) + else: + y = [] + for l in range(0, len(D), self.__N): + y.append(np.array(D[l:l+self.__N])) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)/self.__N))) + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = i/len(minor_index) + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + for r in range(0, len(y)): + clr = next(colors) + self.plot_histogram(ax, y[r], legend_label[0]+'-set-'+str(r)+'_'+str(m), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_DataColsName + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + else: + if self.summary != 'full': + fig, ax = plt.subplots() + if len(dframe.columns) == 2: + y1 = [] + y2 = [] + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 4*len(dframe)/self.__N))) + size = 4*len(dframe)/self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + for r in range(0, len(y1)): # TODO: y1 and y2 length must not be different, add a check + clr = next(colors) + self.plot_histogram(ax, y1[r], legend_label[0]+'_'+str(r), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + clr = next(colors) + self.plot_histogram(ax, y2[r], legend_label[1]+'_'+str(r), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_DataColsName + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "_" + str(file_count) + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + else: + y1 = [] + col_A = dframe[dframe.columns[0]] + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + for r in range(0, len(dframe)//self.__N): + clr = next(colors) + self.plot_histogram(ax, y1[r], legend_label[0]+'_'+str(r), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_DataColsName + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + else: # for the whole ensemble of data, if analysis is not agent level + fig, ax = plt.subplots() + if len(dframe.columns) == 2: ################TODO: two option not needed because when summary full, no custom quantile possible + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 4))) + size = 4 + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + clr = next(colors) + self.plot_histogram(ax, col_A, legend_label[0], clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + clr = next(colors) + self.plot_histogram(ax, col_B, legend_label[1], clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_DataColsName + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "_" + str(file_count) + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + else: + col_A = dframe[dframe.columns[0]] + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 1))) + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.array([0.0]))) + else: + colors = iter(cm.rainbow(np.array([0.0]))) #0.0:blue 1.0:red + + clr = next(colors) + self.plot_histogram(ax, col_A, legend_label[0], clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_DataColsName + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + file_count = file_count + 1 + + def many_output(self): + + if (self.summary == 'full' and self.__analysistype != A.agent): + print(">> Multiple plots not possible for full ensemble of the data, select single plot option instead and retry!") + sys.exit(1) + + step = 1 + if self.summary == 'custom_quantile': + step = 2 + file_count = 0 + for col in range(0, len(self.__data.columns),step): + if self.summary == 'custom_quantile': + dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy().dropna() # one variable, one case at a time + else: + dframe = pd.DataFrame(self.__data[self.__data.columns[col]]).dropna() + + legend_label = dframe.columns + self.__N = len(dframe.index.get_level_values('major').unique()) + + if self.__analysistype == A.agent: + print("- Warning: too many plots will be produced !!! ") + minor_index = dframe.index.get_level_values('minor').unique() # get the index values for minor axis, which will later be used to sort the dataframe + for m in minor_index: + D = dframe.xs( int(m) , level='minor') + if len(D.columns) == 2: + print("Quantile not possible for agent level analysis") + sys.exit(1) + else: + count = 0 + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)//self.__N))) + size = len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + for r in range(0,len(D),self.__N): + fig, ax = plt.subplots() + y = np.array(D[r:r+self.__N]) + clr = next(colors) + self.plot_histogram(ax, y, legend_label[0] + "_run_" + str(count) + "_instance_" + str(m), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #PlotLabel_AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_Count + + #Add a counter to the end + # if count != 0: + # plot_name = str(plot_name) + '_' + str(count) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "_run_" + str(count) + "_" + str(m) + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + count = count + 1 + else: + if len(dframe.columns) == 2: + y1 = [] + y2 = [] + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = 4*len(dframe)//self.__N))) + size = 4*len(dframe)//self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + for r in range(0, len(dframe)//self.__N): + fig, ax = plt.subplots() + clr = next(colors) + self.plot_histogram(ax, y1[r], legend_label[0] + "_run_" + str(r), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + clr = next(colors) + self.plot_histogram(ax, y2[r], legend_label[1] + "_run_" + str(r), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name #PlotName_Count + + #Add a counter to the end + # if file_count != 0: + # plot_name = str(plot_name) + '_' + str(file_count) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + file_count = file_count + 1 + else: + y =[] + for i in range(0,len(dframe),self.__N): + y.append(np.array(dframe[i:i+self.__N])) + + # edit colormap here + #colors = iter(cm.rainbow(np.random.uniform(0, 1, size = len(dframe)/self.__N))) + size = len(dframe)/self.__N + a = np.empty(shape=size,) + for s in range(size): + a[s] = s/size + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(a)) + else: + colors = iter(cm.rainbow(a)) + + for s in range(0, len(dframe)//self.__N): + fig, ax = plt.subplots() + clr = next(colors) + self.plot_histogram(ax, y[s], legend_label[0] + "_" + str(s), clr, self.__P.bins(self.idx), str(self.__data.columns[col])) + + #plot_name + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) #AgentName_DataColsName + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Add a counter to the end + # if file_count != 0: + # plot_name = str(plot_name) + '_' + str(file_count) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + +class Scatterplot(A): + + def __init__(self, idx, data, plt_config, main_param, outpath): + self.idx = idx + self.agent = main_param['agent'] + self.variables = [item for sublist in (list(main_param['variables'].values())) for item in sublist] + self.__data = data + self.__P = plt_config + self.outpath = outpath + '/scatterplot' + self.dir_check(self.outpath) + self.__N = len(main_param['major']) #length of iters list + self.__S = len(main_param['set']) #length of sets list + self.__R = len(main_param['run']) #length of runs list + self.__analysistype = self.map_analysis(main_param['analysis']) + self.analysistype = main_param['analysis'] + self.delay = main_param['delay'] + self.summary = main_param['summary'] + + if self.delay is True: + if self.summary == 'custom_quantile': + print(">> Delay not supported for Quantiles! Adjust parameter and retry!") + sys.exit(1) + delayed_df = self.__data.shift(periods = 1, axis = 0) + delayed_df.rename(columns=lambda x: x+ "_delay", inplace=True) + D = pd.concat([self.__data, delayed_df], axis =1) + self.__data = D[list(sum(zip(delayed_df.columns, self.__data.columns), ()))] + + + def map_analysis(self, val): + analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} + return analysis_values[val] + + # Function to check for existing directories, and create a new one if not present + def dir_check(self, d): + if os.path.exists(d): + print("- Directory ["+os.path.basename(d)+ "] is used for output files") + else: + os.makedirs(d) + print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") + + def plot_scatterplot(self, ax, x, y, l_label, xlabel, ylabel, clr): + if self.__P.legend_label(self.idx) is None: + le_label = l_label + else: + le_label = self.__P.legend_label(self.idx) + + out = ax.scatter(x, y, linestyle=self.__P.linestyle(self.idx), marker=self.__P.marker(self.idx), + facecolor=self.__P.facecolors(self.idx), label=le_label, color=clr) + + if self.__P.legend(self.idx) is True: + ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) + + #plt.xlabel(xlabel) + #plt.ylabel(ylabel) + + #Axes labels + if self.__P.xlabel(self.idx) is None or self.__P.xlabel(self.idx) == 'Auto' or self.__P.xlabel(self.idx) == 'auto': + ax.set_xlabel(str(self.agent) + ' : ' + str(xlabel)) + else: + ax.set_xlabel(self.__P.xlabel(self.idx)) + + if self.__P.ylabel(self.idx) is None or self.__P.ylabel(self.idx) == 'Auto' or self.__P.ylabel(self.idx) == 'auto': + ax.set_ylabel(str(self.agent) + ' : ' + str(ylabel)) + else: + ax.set_ylabel(self.__P.ylabel(self.idx)) + + #No axes labels + if self.__P.xlabel(self.idx) == 'None' or self.__P.xlabel(self.idx) == 'none': + ax.set_xlabel("") + + if self.__P.ylabel(self.idx) == 'None' or self.__P.ylabel(self.idx) == 'none': + ax.set_ylabel("") + + plt.axis([self.__P.xmin(self.idx),self.__P.xmax(self.idx),self.__P.ymin(self.idx),self.__P.ymax(self.idx)]) + + return out + + def one_plot(self): + file_count = 0 + step = 2 + + for col in range(0, len(self.__data.columns), step): + if len(self.__data.columns) < 2: + print(">> Problem with data! Either set delay to True, or specify at least two variables to plot!") + sys.exit(1) + + dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() + + #test + # print('\nPrint (plot.py): self.__data') + # print(self.__data) + # print('\nPrint (plot.py): dframe') + # print(dframe) + + # scatter one_plot agent + if self.__analysistype == A.agent: + minor_index = dframe.index.get_level_values('minor').unique() + fig, ax = plt.subplots() # initialize figure + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.linspace(0, 1, len(dframe)//self.__N))) + else: + colors = iter(cm.rainbow(np.linspace(0, 1, len(dframe)//self.__N))) + + #Using groupby on minor index to re-order hierarchical index + grouped = dframe.groupby('minor') + # print(grouped.get_group(0)) + # print(grouped.get_group(1)) + + #m: agent index (not IDs) + for m in minor_index: + + #Select only agent index m + D = grouped.get_group(m) + + ##Test + # print('D = grouped.get_group('+str(m)+')') + # print(D) + + #Old code refactored: + #D = dframe.xs(int(m), level='minor') #agent-specific dframe containing: set,run,iter,vars + + #test + # print('\nPrint (plot.py): D') + # print(D) + + legend_label = D.columns #legend_label: list of variable names + if len(dframe.columns) != 2: + print(">> Something wrong with data, check and retry!") + sys.exit (1) + + #Refactor candidate: + y1 = [] + y2 = [] + col_A = D[D.columns[0]] #col_A: values of variable 0 + col_B = D[D.columns[1]] #col_B: values of variable 1 + + ##Test + # print('\nPrint (plot.py): col_A') + # print(col_A) + # print('\nPrint (plot.py): col_B') + # print(col_B) + + #Refactor candidate: Series of Array for all set+run data + #Append set_run data for agent index m + # Note: range(0, len(D), self.__N) uses as increment self.__N, + # so it is selecting rows of D that are self.__N rows apart. + for i in range(0, len(D), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + + ##Test + # print('\nPrint (plot.py): y1') + # print(y1) + # print('\nPrint (plot.py): y2') + # print(y2) + + # Index values: + print("D.index.levels:") + #print(D.index.levels) #D only contains sets, runs, iters, no agents + + print("sets:") + print('D.index.levels[0].values') + print(D.index.levels[0].values) #set values + + print("runs:") + print('D.index.levels[1].values') + print(D.index.levels[1].values) #run values + # print("iters:") + # print('D.index.levels[2].values') + # print(D.index.levels[2].values) #iter values + + nsets = len(D.index.levels[0]) + nruns = len(D.index.levels[1]) + + print("Length set list:") + print(nsets) + + print("Length run list:") + print(nruns) + + for ind, r in enumerate(range(0, len(D)//self.__N)): + ## print('Loop dframe block r: '+str(ind)+'/'+str(len(D)//self.__N)) + + clr = next(colors) + + # Index values: block in r-loop + run_idx = r % nruns # r MOD blocks, ex: r MOD 4 + set_idx = r // nruns # r DIV blocks, ex: r DIV 4 + + #TEST + # print("set:") + # print(D.index.levels[0].values[set_idx]) + + # print("run:") + # print(D.index.levels[1].values[run_idx]) + + ##print("iters:") + ##print(D.index.levels[2].values) + + #Agent index: does not exist in D, but does in dframe + #TODO: use dframe or d or df_main (all not defined n this function) + # print("agent index:") + # print(m) + + #Use run and set labels derived from row index r + set_no = D.index.levels[0].values[set_idx] + run_no = D.index.levels[1].values[run_idx] + + #TODO: Use agent label derived from agent index m (required access to global dataframe) + agent_ind = m + #agent_id = df_main.index.levels[3].values[m] #TODO + + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ' run ' + str(run_no) + ' agent '+ str(agent_ind) + ']', legend_label[0], legend_label[1], clr) + + #Scatter plot_name one_plot agent case 0 + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Test code + #if args.trace: + print("Scatterplot [case 0 one_plot analysis=Agent]: ") #agent analysis, one_plot + print(plot_name) + #End of Test code + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + else: # scatter one_plot summary!='full' multiple_run,multiple_batch,multiple_set + if self.summary != 'full': + fig, ax = plt.subplots() # initialize figure + legend_label = dframe.columns + if len(dframe.columns) != 2: + print(">> Something wrong with data, check and retry!") + sys.exit (1) + y1 = [] + y2 = [] + + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + ##Test + # print('\nPrint (plot.py): col_A') + # print(col_A) + # print('\nPrint (plot.py): col_B') + # print(col_B) + + #Refactor candidate: Series of Array for all set+run data + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.linspace(0, 1, len(y1)))) + else: + colors = iter(cm.rainbow(np.linspace(0, 1, len(y1)))) + + # Block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, summary across runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + #print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, summary across batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + for ind, r in enumerate(range(0, len(dframe)//self.__N)): + ## print('Loop dframe block r: '+str(ind)+'/'+str(len(dframe)//self.__N)) + + clr = next(colors) + + # Block inside r-loop: set_no run_no + if(self.analysistype == 'multiple_run'): + run_idx = r % nruns + set_idx = r // nruns + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values[run_idx] + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ' run ' + str(run_no) + ']', legend_label[0], legend_label[1], clr) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("run:", dframe.index.levels[1].values[run_idx]) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + if(self.analysistype == 'multiple_batch'): + #run_idx = r % nruns #irrelevant? + set_idx = r + set_no = dframe.index.levels[0].values[set_idx] + run_no = 0 + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ' ' + self.summary + ']', legend_label[0], legend_label[1], clr) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("iters:", dframe.index.levels[1].values) #run values + if(self.analysistype == 'multiple_set'): + set_no = self.summary + run_no = 0 + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ']', legend_label[0], legend_label[1], clr) + #print("iters:", dframe.index.values) #iter values + + #Scatter plot_name one_plot not-agent not-full case 1 + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Test + #if args.trace: + print("Scatterplot [case 1 one_plot, analysis != Agent, summary != 'full']: ") #multiple_batch + print(plot_name) + #End of Test code + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + else: # scatter one_plot summary=='full' multiple_run,multiple_batch,multiple_set + fig, ax = plt.subplots() # initialize figure + legend_label = dframe.columns + if len(dframe.columns) != 2: + print(">> Something wrong with data, check and retry!") + sys.exit (1) + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.linspace(0, 1, 1))) + else: + colors = iter(cm.rainbow(np.linspace(0, 1, 1))) + + clr = next(colors) + + # block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, full runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + # print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, full batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) #iter values + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + if(self.analysistype == 'multiple_run'): + set_no = dframe.index.levels[0].values #names + run_no = dframe.index.levels[1].values #names + self.plot_scatterplot(ax, col_A, col_B, '[set ' + str(set_no) + ' run ' + str(run_no) + ']', legend_label[0], legend_label[1], clr) + if(self.analysistype == 'multiple_batch'): + set_no = dframe.index.levels[0].values #names + run_no = dframe.index.levels[1].values #names + self.plot_scatterplot(ax, col_A, col_B, '[set ' + str(set_no) + ' run ' + str(run_no) + ']', legend_label[0], legend_label[1], clr) + if(self.analysistype == 'multiple_set'): + set_no = self.summary + run_no = 0 + self.plot_scatterplot(ax, col_A, col_B, '[set ' + str(set_no) + ']', legend_label[0], legend_label[1], clr) + + #Scatter plot_name one_plot not-agent full case 2 + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + '_full' + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #if args.trace: + print("Scatterplot [case 2 one_plot, analysis != Agent, summary = 'full']:") + print(plot_name) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + file_count = file_count + 1 + + def many_output(self): + file_count = 0 + step = 2 + for col in range(0, len(self.__data.columns), step): + if len(self.__data.columns) < 2: + print("Problem with data! Either set delay to True, or specify at least two variables to plot!") + sys.exit(1) + dframe = self.__data[[self.__data.columns[col], self.__data.columns[col+1]]].copy() + + # scatter many_plot agent + if self.__analysistype == A.agent: + minor_index = dframe.index.get_level_values('minor').unique() + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.linspace(0, 1, len(dframe)//self.__N))) + else: + colors = iter(cm.rainbow(np.linspace(0, 1, len(dframe)//self.__N))) + + #Use groupby + grouped = dframe.groupby('minor') + + for m in minor_index: + D = grouped.get_group(m) + #D = dframe.xs(int(m), level='minor') # remove + legend_label = D.columns + if len(dframe.columns) != 2: + print("Something wrong with data, check and retry!") + sys.exit (1) + y1 = [] + y2 = [] + col_A = D[D.columns[0]] + col_B = D[D.columns[1]] + for i in range(0, len(D), self.__N): # blocks of self.__N rows belong to one agent + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + + nsets = len(D.index.levels[0]) + nruns = len(D.index.levels[1]) + + for ind, r in enumerate(range(0, len(D)//self.__N)): # loop all rows/no.iters + ## print('Loop dframe block r: '+str(ind)+'/'+str(len(D)//self.__N)) + fig, ax = plt.subplots() #open new figure for many plot option + clr = next(colors) + + # Index values: + run_idx = r % nruns # r MOD blocks, ex: r MOD 4 + set_idx = r // nruns # r DIV blocks, ex: r DIV 4 + + #Use run and set labels derived from row index r + set_no = D.index.levels[0].values[set_idx] + run_no = D.index.levels[1].values[run_idx] + + #TODO: Use agent label derived from agent index m (required access to global dataframe) + agent_ind = m + #agent_id = df_main.index.levels[3].values[m] #TODO + + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ' run ' + str(run_no) + ' agent '+ str(agent_ind) + ']', legend_label[0], legend_label[1], clr) + + ##Scatter plot_name many_output agent case 3 + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + '_set_' + str(set_no) + '_run_' + str(run_no) + '_' + 'agent_'+ str(agent_ind) + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Add a counter to the end + # if file_count != 0: + # plot_name = str(plot_name) + '_' + str(file_count) + + #if args.trace: + print("Scatterplot [case 3 many_output analysis=Agent]:") + print(plot_name) + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + else: # scatter many_plot summary!='full' multiple_run,multiple_batch,multiple_set + if self.summary != 'full': + legend_label = dframe.columns + if len(dframe.columns) != 2: + print(">> Something wrong with data, check and retry!") + sys.exit (1) + y1 = [] + y2 = [] + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + ##Test + # print('\nPrint (plot.py): col_A') + # print(col_A) + # print('\nPrint (plot.py): col_B') + # print(col_B) + + #Refactor candidate: Series of Array for all set+run data + #This block is only needed if summary != 'full' + for i in range(0, len(dframe), self.__N): + y1.append(np.array(col_A[i:i+self.__N])) + y2.append(np.array(col_B[i:i+self.__N])) + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.linspace(0, 1, len(y1)))) + else: + colors = iter(cm.rainbow(np.linspace(0, 1, len(y1)))) + + # block: nsets nruns + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, summary across runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + #print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, summary across batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + # many plot + for ind, r in enumerate(range(0, len(dframe)//self.__N)): + ## print('Loop dframe block r: '+str(ind)+'/'+str(len(dframe)//self.__N)) + fig, ax = plt.subplots() #open new figure for many plot + clr = next(colors) + + if(self.analysistype == 'multiple_run'): + run_idx = r % nruns + set_idx = r // nruns + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values[run_idx] + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ' run ' + str(run_no) + ']', legend_label[0], legend_label[1], clr) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("run:", dframe.index.levels[1].values[run_idx]) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + if(self.analysistype == 'multiple_batch'): + #run_idx = r % nruns #Irrelevant? + set_idx = r % nsets + set_no = dframe.index.levels[0].values[set_idx] + run_no = dframe.index.levels[1].values #names + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ']', legend_label[0], legend_label[1], clr) + # print("set:", dframe.index.levels[0].values[set_idx]) #set values + # print("iters:", dframe.index.levels[1].values) #run values + if(self.analysistype == 'multiple_set'): + #run_idx = r % nruns #Irrelevant? + #set_idx = r // nruns + set_no = self.summary + run_no = 0 + self.plot_scatterplot(ax, y1[r], y2[r], '[set ' + str(set_no) + ']', legend_label[0], legend_label[1], clr) + #print("iters:", dframe.index.values) #iter values + + ##Scatter plot_name many_output not-agent not-full case 4 + #Note: this block is indented directly under plot_scatterplot, in order to have many plots + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + '_set_' + str(set_no) + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Add a counter to the end + if file_count != 0: + plot_name = str(plot_name) + '_' + str(file_count) + + #if args.trace: + print("Scatterplot [case 4 many_output, analysis != Agent]:") + print(plot_name) + #print("file_count= "+str(file_count)) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + else: # scatter many_plot summary=='full' multiple_run,multiple_batch,multiple_set + fig, ax = plt.subplots() #open new figure for many plot + + legend_label = dframe.columns + if len(dframe.columns) != 2: + print(">> Something wrong with data, check and retry!") + sys.exit (1) + col_A = dframe[dframe.columns[0]] + col_B = dframe[dframe.columns[1]] + + ##Test + # print('\nPrint (plot.py): col_A') + # print(col_A) + # print('\nPrint (plot.py): col_B') + # print(col_B) + + # edit colormap here + if self.__P.greyscale(self.idx): + colors = iter(cm.gray(np.linspace(0, 1, 1))) + else: + colors = iter(cm.rainbow(np.linspace(0, 1, 1))) + + clr = next(colors) + + if(self.analysistype == 'multiple_run'): + print("Analysis type: multiple_run") + print("Description: Show all runs, summary across agents") + nsets = len(dframe.index.levels[0]) + nruns = len(dframe.index.levels[1]) + niter = len(dframe.index.levels[2]) + print("sets:", dframe.index.levels[0].values) #set values + print("runs:", dframe.index.levels[1].values) #run values + # print("iters:", dframe.index.levels[2].values) #iter values + + if(self.analysistype == 'multiple_batch'): + print("Analysis type: multiple_batch") + print("Description: Show all batches, full runs") + nsets = len(dframe.index.levels[0]) + nruns = 0 + niter = len(dframe.index.levels[1]) + print("sets:", dframe.index.levels[0].values) #set values + # print("iters:", dframe.index.levels[1].values) #iter values + + if(self.analysistype == 'multiple_set'): + print("Analysis type: multiple_set") + print("Description: Show all sets, full batches") + nsets = 1 + nruns = 0 + niter = len(dframe) + #print("iters:", dframe.index.values) #iter values + + print("Length set list:", nsets) + print("Length run list:", nruns) + print("Length iter list:", niter) + + if(self.analysistype == 'multiple_run'): + set_no = dframe.index.levels[0].values #names + run_no = dframe.index.levels[1].values #names + self.plot_scatterplot(ax, col_A, col_B, '[set ' + str(set_no) + ' run ' + str(run_no) + ']', legend_label[0], legend_label[1], clr) + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + '_sets_' + str(nsets) + '_runs_' + str(nruns) + if(self.analysistype == 'multiple_batch'): + set_no = dframe.index.levels[0].values #names + run_no = "" #dframe.index.levels[1].values #names + self.plot_scatterplot(ax, col_A, col_B, '[set ' + str(set_no) + ' run ' + str(run_no) + ']', legend_label[0], legend_label[1], clr) + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + '_sets_' + str(nsets) + if(self.analysistype == 'multiple_set'): + set_no = self.summary + run_no = "" #0 + self.plot_scatterplot(ax, col_A, col_B, '[set ' + str(set_no) + ']', legend_label[0], legend_label[1], clr) + plot_name = str(self.agent) + '_' + str("_".join(self.variables)) + '_sets_' + str(nsets) + + ##Scatter plot_name many_output not-agent summary-full case 5 + #Note: this block is indented directly under plot_scatterplot, in order to have many plots + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + #Add a counter to the end + # if file_count != 0: + # plot_name = str(plot_name) + '_' + str(file_count) + + #if args.trace: + print("Scatterplot [case 5 many_output, analysis != Agent, summary==full]:") + print(plot_name) + #print("file_count= "+str(file_count)) + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + file_count = file_count + 1 + +class Boxplot(A): + def __init__(self, idx, data, plt_config, main_param, outpath): + self.idx = idx + self.agent = main_param['agent'] + self.__data = data + self.__P = plt_config + self.__main_param = main_param + self.outpath = outpath + '/boxplot' + self.dir_check(self.outpath) + self.__N = len(main_param['major']) + self.__analysistype = self.map_analysis(main_param['analysis']) + self.analysistype = main_param['analysis'] + if self.__analysistype == A.agent: + print("Boxplot not possible for agent-level analysis!") + sys.exit(1) + + def map_analysis(self, val): + + analysis_values = {'agent' : A.agent, 'multiple_run' : A.multiple_run, 'multiple_batch' : A.multiple_batch, 'multiple_set' : A.multiple_set} + return analysis_values[val] + + # Function to check for existing directories, and create a new one if not present + def dir_check(self, d): + if os.path.exists(d): + print("- Directory ["+os.path.basename(d)+ "] is used for output files") + else: + os.makedirs(d) + print("- Directory ["+os.path.basename(d)+ "] was created and is used for output files") + + def process_boxplot_data(self, data): + + s = SummaryStats(data, self.__main_param) + box_df = pd.DataFrame() + #print s.mean().values + box_df['mean'] = s.mean() + box_df['median'] = s.median() + box_df['third_quartile'] = s.third_quartile() + box_df['first_quartile'] = s.first_quartile() + box_df['max'] = s.maximum() + box_df['min'] = s.minimum() + return box_df + + #def plot_boxplot(self, ax, data, l_label): + def plot_boxplot(self, ax, data, l_label, forced_ylabel): + + if self.__P.legend_label(self.idx) is None: + le_label = l_label + else: + le_label = self.__P.legend_label(self.idx) + + t_df = data.T + intervals = [] + pos = [] + N_bins = self.__P.bins(self.idx) + count = 1 + for i in range(0,len(self.__main_param['major']), int(np.floor(len(self.__main_param['major'])//N_bins))): + intervals.append(self.__main_param['major'][i]) + pos.append(count) + count = count + 1 + #ax = t_df.boxplot(column = [100,250,500,750,900], positions =[1,2,3,4,5]) + #ax = t_df.boxplot(column = intervals, positions =pos) + + # edit colormap here + if self.__P.greyscale(self.idx): + color = dict(boxes='Gray', whiskers='Black', medians='Black', caps='Black') + else: + color = dict(boxes='LightGreen', whiskers='Black', medians='Red', caps='Black') + + # plot the boxplots with pandas + df = pd.DataFrame(t_df, columns=intervals) + ax = df.plot.box(by=pos, color=color, patch_artist=True) + + if self.__P.plot_title(self.idx) is not None: + ax.set_title(self.__P.plot_title(self.idx)) + + #Axes labels + if self.__P.xlabel(self.idx) is None or self.__P.xlabel(self.idx) == 'Auto' or self.__P.xlabel(self.idx) == 'auto': + ax.set_xlabel("Time") + else: + ax.set_xlabel(self.__P.xlabel(self.idx)) + + if self.__P.ylabel(self.idx) is None or self.__P.ylabel(self.idx) == 'Auto' or self.__P.ylabel(self.idx) == 'auto': + ax.set_ylabel(str(self.agent) + ' : ' + forced_ylabel) + else: + ax.set_ylabel(self.__P.ylabel(self.idx)) + + #No axes labels + if self.__P.xlabel(self.idx) == 'None' or self.__P.xlabel(self.idx) == 'none': + ax.set_xlabel("") + + if self.__P.ylabel(self.idx) == 'None' or self.__P.ylabel(self.idx) == 'none': + ax.set_ylabel("") + + plt.axis([self.__P.xmin(self.idx),self.__P.xmax(self.idx),self.__P.ymin(self.idx),self.__P.ymax(self.idx)]) + + #TODO: Legend for boxplots + # Use plt (matplotlib) instead of ax (pandas)? ax.set_legend does not exist + #if self.__P.legend(self.idx) is True: + # plt.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) + if self.__P.legend(self.idx) is True: + ax.legend(loc=self.__P.legend_location(self.idx), fancybox=True, shadow=True) + + return ax + + def one_plot(self): + + for col in range(0, len(self.__data.columns)): + dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) + fig, ax = plt.subplots() + #if self.__analysistype == A.agent: # check done above, redundant, to remove + # print("Boxplot not possible for agent-level analysis!") + # sys.exit(1) + #else: + col_A = dframe[dframe.columns[0]] + D = self.process_boxplot_data(col_A) + y = [] + for i in range(0, len(D), self.__N): + y.append(pd.DataFrame(D[i:i+self.__N])) + + for r in range(0, len(D)//self.__N): + self.plot_boxplot(ax, y[r], self.__data.columns[col], str(self.__data.columns[col])) + + #Boxplots plot_name one_plot case 1 + plot_name = str(self.agent) + '_' + str(self.__data.columns[col]) + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() + + def many_output(self): + + for col in range(0, len(self.__data.columns)): + dframe = pd.DataFrame(self.__data[self.__data.columns[col]]) + #if self.__analysistype == A.agent: # check done above, redundant, to remove + # print("Boxplot not possible for agent-level analysis!") + # sys.exit(1) + #else: + col_A = dframe[dframe.columns[0]] + D = self.process_boxplot_data(col_A) + y =[] + for i in range(0,len(D),self.__N): + y.append(pd.DataFrame(D[i:i+self.__N])) + for s in range(0, len(D)//self.__N): + fig, ax = plt.subplots() + self.plot_boxplot(ax, y[s], self.__data.columns[col], str(self.__data.columns[col])) + + #Boxplots plot_name many_output case 2 + plot_name = str(self.idx) + '_' + str(self.agent) + '_' + str(self.__data.columns[col]) + if self.__P.plot_name(self.idx): + plot_name = str(self.__P.plot_name(self.idx)) + '_' + plot_name + + plot_format = self.__P.plot_format(self.idx) + plt.savefig(self.outpath + '/' + plot_name + "." + plot_format, format=plot_format, bbox_inches='tight') + plt.close() diff --git a/TODO/visualisation/src/summarystats.py b/src/visualisation_scripts/summarystats.py similarity index 86% rename from TODO/visualisation/src/summarystats.py rename to src/visualisation_scripts/summarystats.py index fee8028..1d92446 100644 --- a/TODO/visualisation/src/summarystats.py +++ b/src/visualisation_scripts/summarystats.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import pandas as pd from parameters import A # imported self written class from directory @@ -25,7 +25,7 @@ def __init__(self, data, param): #print self.__data.head(5) def compute_summary(self): - summary_type = {'mean': self.mean, 'median': self.median, 'upper_quartile': self.upper_quartile, 'lower_quartile': self.lower_quartile, 'custom_quantile': self.custom_quantile, 'minimum': self.minimum, 'maximum': self.maximum, 'full':self.full} + summary_type = {'mean': self.mean, 'median': self.median, 'third_quartile': self.third_quartile, 'first_quartile': self.first_quartile, 'custom_quantile': self.custom_quantile, 'minimum': self.minimum, 'maximum': self.maximum, 'full':self.full} return summary_type[self.__param['summary']]() def map_analysis(self): @@ -61,7 +61,9 @@ def custom_quantile(self): Q = float(N[0]) s = s.append(self.quantile(Q)) for i in range(len(list(s))): - s.rename(columns={list(s)[i]: str(list(s)[i])+"_quantile ("+str(Q)+")"}, inplace=True) + #s.rename(columns={list(s)[i]: str(list(s)[i])+"_quantile ("+str(Q)+")"}, inplace=True) + s.rename(columns={list(s)[i]: str(list(s)[i])+"_Q("+str(Q)+")"}, inplace=True) + #s.rename(columns={list(s)[i]: "Q("+str(Q)+")"}, inplace=True) # quantile label: Q(0.2) return s elif len(N) == 2: @@ -71,13 +73,15 @@ def custom_quantile(self): Q2 = float(N[1]) s1 = s1.append(self.quantile(Q1)) # data frame for lower quantile for i in range(len(list(s1))): - s1.rename(columns={list(s1)[i]: str(list(s1)[i])+"_l_quantile ("+str(Q1)+")"}, inplace=True) + s1.rename(columns={list(s1)[i]: str(list(s1)[i])+"_Q("+str(Q1)+")"}, inplace=True) # varname_Q(0.2) + # s1.rename(columns={list(s1)[i]: "Q("+str(Q1)+")"}, inplace=True) # lower quantile label: Q(0.2) s2 = s2.append(self.quantile(Q2)) # data frame for upper quantile for i in range(len(list(s2))): - s2.rename(columns={list(s2)[i]: str(list(s2)[i])+"_u_quantile ("+str(Q2)+")"}, inplace=True) - - D = pd.concat([s1, s2], axis=1) + s2.rename(columns={list(s2)[i]: str(list(s2)[i])+"_Q("+str(Q2)+")"}, inplace=True) # varname_Q(0.8) + # s2.rename(columns={list(s2)[i]: "Q("+str(Q2)+")"}, inplace=True) # upper quantile label: Q(0.8) + + D = pd.concat([s1, s2], axis=1) return D[list(sum(zip(s1.columns, s2.columns), ()))] # arrange columns (l_q, u_q) in alternating fashion def maximum(self): @@ -101,8 +105,8 @@ def minimum(self): def median(self): return self.quantile(0.50) - def upper_quartile(self): + def third_quartile(self): return self.quantile(0.75) - def lower_quartile(self): + def first_quartile(self): return self.quantile(0.25) diff --git a/TODO/visualisation/src/transform.py b/src/visualisation_scripts/transform.py similarity index 59% rename from TODO/visualisation/src/transform.py rename to src/visualisation_scripts/transform.py index 4d20d8f..f2506c3 100644 --- a/TODO/visualisation/src/transform.py +++ b/src/visualisation_scripts/transform.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys, os import yaml import numpy as np @@ -9,21 +9,21 @@ class Transform(): def __init__(self, idx, data, par_fpath): - self.__idx = idx + self.__idx = idx self.__data = data self.P = transform_configuration(par_fpath) self.df_out = pd.DataFrame(data=None, columns=self.__data.columns,index=self.__data.index) - print("lau laus") + print("- Transformation") - def main_method(self, outpath): - transform_function = {'qoq_annual_frequency': self.qoq_annual_frequency, 'qoq_quaterly_frequency': self.qoq_quaterly_frequency, 'mom_annual_frequency': self.mom_annual_frequency, 'mom': self.mom,'annual_P_I_T': self.annual_P_I_T} + def main_method(self, outpath): + transform_function = {'qoq_annual': self.qoq_annual, 'mom_annual': self.mom_annual, 'mom': self.mom, 'pct_change_yoy': self.pct_change_yoy, 'pct_change_qoq': self.pct_change_qoq} fn = self.P.get_parameters(self.__idx)['aggregate'] data_out = transform_function[self.P.get_parameters(self.__idx)['transform_function']](fn, self.df_out) f_out = self.P.get_parameters(self.__idx)['write_file'] if f_out is True: data_out.to_hdf(str(outpath)+ '/' + str(self.P.get_parameters(self.__idx)['output_file_name']), str(self.P.get_parameters(self.__idx)['hdf_groupname']), mode = 'a', format = 'table') - + return data_out @@ -36,84 +36,85 @@ def col_name_mapper(self): # map old var name with transformed var name return d - def qoq_annual_frequency(self,fn, d_out): # method to print quaterly growth rate (quarter on quarter) + def qoq_annual(self,fn, d_out): # method to print quarterly growth rate (quarter on same quarter of previous year) variables = self.P.get_parameters(self.__idx)['variables'] col_d = self.col_name_mapper() - def mean(df_out): # TODO: value seems wrong, check, multi agent causing error - roll_mean = self.__data[variables.values()].rolling(window=3, min_periods=3).mean() # first get rolling window values with step 3 and initial buffer 3 - #df_out[variables.values()] = roll_mean[::3].pct_change(4) + def mean(df_out): # TODO: value seems wrong, check, multi agent causing error + roll_mean = self.__data[variables.values()].rolling(window=3, min_periods=3).mean() # first get rolling window values with step 3 and initial buffer 3 + #df_out[variables.values()] = roll_mean[::3].pct_change(4) df_out[variables.values()] = roll_mean[::3][variables.values()] - df_out = df_out.pct_change(4) - + df_out = df_out.pct_change(4) # compute pct change with respect to same observation (quarter) 4 periods ago return df_out.rename(columns = col_d) - + def summation(df_out): roll_sum = self.__data[variables.values()].rolling(window=3,min_periods=3).sum() df_out[variables.values()] = roll_sum[::3].pct_change(4) - return df_out.rename(columns = col_d) + return df_out.rename(columns = col_d) - f_mapper = {'mean': mean, 'sum': summation} # map the function to apply with the desired input + f_mapper = {'mean': mean, 'sum': summation} # map the function to apply with the desired input return f_mapper[fn](d_out) - def mom_annual_frequency(self,fn, df_out): # method to print monthly growth rate (month on month) + def mom_annual(self,fn, df_out): # method to print monthly growth rate (month on same month previous year) variables = self.P.get_parameters(self.__idx)['variables'] col_d = self.col_name_mapper() - df_out[variables.values()] = self.__data[variables.values()].pct_change(12) + df_out[variables.values()] = self.__data[variables.values()].pct_change(12) # compute pct change with respect to same observation (month) 12 periods ago return df_out.rename(columns = col_d) - def mom(self,fn, df_out): # method to print monthly growth rate (month on month in one cycle) - + def mom(self,fn, df_out): # method to print monthly growth rate (month on month) variables = self.P.get_parameters(self.__idx)['variables'] col_d = self.col_name_mapper() - df_out[variables.values()] = self.__data[variables.values()].pct_change(1) + df_out[variables.values()] = self.__data[variables.values()].pct_change(1) # compute pct change with respect to 1 period ago return df_out.rename(columns = col_d) - - def annual_P_I_T(self,fn, df_out): # method to print annual growth rate, point in time. TODO: point in time not added yet - + # method to compute percent change YoY (year-on-year), based on either + # 12-month trailing avg: pct_change_yoy.mean, or + # 12-month trailing sum: pct_change_yoy.sum + def pct_change_yoy(self,fn, df_out): variables = self.P.get_parameters(self.__idx)['variables'] col_d = self.col_name_mapper() def mean(): - roll_mean = self.__data[variables.values()].rolling(window=12,min_periods=12).mean() # first get rolling window values with step 12 and initial buffer 12 - df_out[variables.values()] = roll_mean[::12].pct_change(1) # compute rate between values with a step size 1 + roll_mean = self.__data[variables.values()].rolling(window=12,min_periods=12).mean() # 12-month trailing average: first get rolling window values with step 12 and initial buffer 12 + df_out[variables.values()] = roll_mean[::12].pct_change(1) # pct_change_yoy.mean: compute rate between values with a step size 1: ratio of two consequetive 12-month trailing averages return df_out.rename(columns = col_d) - + def summation(): - roll_sum = self.__data[variables.values()].rolling(window=12,min_periods=12).sum() - df_out[variables.values()] = roll_sum[::12].pct_change(1) + roll_sum = self.__data[variables.values()].rolling(window=12,min_periods=12).sum() # 12-month trailing sum: first get rolling window values with step 12 and initial buffer 12 + df_out[variables.values()] = roll_sum[::12].pct_change(1) # pct_change_yoy.sum: compute rate between values with a step size 1: ratio of two consequetive 12-month trailing sums return df_out.rename(columns = col_d) - - f_mapper = {'mean': mean, 'sum': summation} # map the function to apply with the desired input + f_mapper = {'mean': mean, 'sum': summation} # map the function to apply with the desired input return f_mapper[fn]() - - def qoq_quaterly_frequency(self,fn, df_out): # method to print quaterly growth rate (quarter on quarter) + # method to compute percent change QoQ (quarter-on-quarter), based on either + # 3-month trailing avg: pct_change_qoq.mean, or + # 3-month trailing sum: pct_change_qoq.sum + def pct_change_qoq(self,fn, df_out): variables = self.P.get_parameters(self.__idx)['variables'] col_d = self.col_name_mapper() def mean(): - roll_mean = self.__data[variables.values()].rolling(window=3,min_periods=3).mean() # first get rolling window values with step 3 and initial buffer 3 - df_out[variables.values()] = roll_mean[::3].pct_change(1) + roll_mean = self.__data[variables.values()].rolling(window=3,min_periods=3).mean() # 3-month trailing average: first get rolling window values with step 3 and initial buffer 3 + df_out[variables.values()] = roll_mean[::3].pct_change(1) # pct_change_qoq.mean: compute rate between values with a step size 1: ratio of two consequetive 3-month trailing averages return df_out.rename(columns = col_d) - + def summation(): - roll_sum = self.__data[variables.values()].rolling(window=3,min_periods=3).sum() - df_out[variables.values()] = roll_sum[::3].pct_change(1) + roll_sum = self.__data[variables.values()].rolling(window=3,min_periods=3).sum() # 3-month trailing sum: first get rolling window values with step 3 and initial buffer 3 + df_out[variables.values()] = roll_sum[::3].pct_change(1) # pct_change_qoq.sum: compute rate between values with a step size 1: ratio of two consequetive 3-month trailing sums return df_out.rename(columns = col_d) - - f_mapper = {'mean': mean, 'sum': summation} # map the function to apply with the desired input + + f_mapper = {'mean': mean, 'sum': summation} # map the function to apply with the desired input return f_mapper[fn]() if __name__ == "__main__": - C = transform_configuration('/home/susupta/Desktop/Fix_preprocess/visualize/src') + C = transform_configuration() print(C.get_parameters()['variables']) - +## Notes: +## Seems this script is still incomplete \ No newline at end of file