From 929d6dff367b20e075bb99f7b0733210bb7952c3 Mon Sep 17 00:00:00 2001 From: Peter Vorman Date: Mon, 9 Feb 2026 23:03:43 +0200 Subject: [PATCH 1/2] init --- .coveragerc | 12 + LICENSE | 661 ++++++++++++++++++ README.md | 53 ++ ckanext/__init__.py | 7 + ckanext/consumer/__init__.py | 0 ckanext/consumer/consumer.py | 132 ++++ ckanext/consumer/interfaces.py | 26 + ckanext/consumer/plugin.py | 55 ++ ckanext/tests/conftest.py | 113 +++ .../tests/test_consumer_get_kafka_config.py | 41 ++ .../tests/test_consumer_process_message.py | 79 +++ ckanext/tests/test_consumer_run_consumer.py | 91 +++ ckanext/tests/test_interfaces.py | 5 + ckanext/tests/test_plugin.py | 73 ++ dev-requirements.txt | 3 + pytest.ini | 3 + requirements.txt | 1 + setup.py | 28 + 18 files changed, 1383 insertions(+) create mode 100644 .coveragerc create mode 100644 LICENSE create mode 100644 README.md create mode 100644 ckanext/__init__.py create mode 100644 ckanext/consumer/__init__.py create mode 100644 ckanext/consumer/consumer.py create mode 100644 ckanext/consumer/interfaces.py create mode 100644 ckanext/consumer/plugin.py create mode 100644 ckanext/tests/conftest.py create mode 100644 ckanext/tests/test_consumer_get_kafka_config.py create mode 100644 ckanext/tests/test_consumer_process_message.py create mode 100644 ckanext/tests/test_consumer_run_consumer.py create mode 100644 ckanext/tests/test_interfaces.py create mode 100644 ckanext/tests/test_plugin.py create mode 100644 dev-requirements.txt create mode 100644 pytest.ini create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..07f18f6 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,12 @@ +[run] +branch = True +source = + ckanext/reactor + +[report] +show_missing = True +skip_covered = False +fail_under = 90 +exclude_lines = + pragma: no cover + if __name__ == .__main__.: diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0ad25db --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f6e505c --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# ckanext-consumer + +**Event-driven architecture extension for CKAN.** + +`ckanext-consumer` allows your CKAN instance to react to external events. It listens to a Kafka queue, consumes CloudEvents (or standard JSON), and dispatches them to registered Python functions in your other CKAN extensions. + +## 🏗 Architecture + +1. **Consumer** acts as the worker/consumer process (`ckan consumer consume`). +2. It discovers other plugins implementing the `IConsumer` interface. +3. It routes incoming Kafka messages to the correct plugin based on the **Topic Name**. + +## 🔧 Installation + +1. **Install the extension:** + ```bash + pip install -e . + ``` + +2. **Install dependencies:** + ```bash + pip install -r requirements.txt + ``` + +3. **Enable the plugin** in your `ckan.ini`: + ```ini + ckan.plugins = ... consumer + ``` + +## ⚙️ Configuration + +Add the following settings to your `ckan.ini` file. + +### Basic Configuration +ckan.consumer.kafka.bootstrap.servers = your-kafka-broker:9092 +ckan.consumer.kafka.group_id = ckan_consumer_prod +ckan.consumer.kafka.client.id = ckan_instance_1 + +### Security & Authentication (SASL/SSL) +# Protocol: PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL +ckan.consumer.kafka.security.protocol = SASL_SSL + +# Mechanism: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, GSSAPI (Kerberos) +ckan.consumer.kafka.sasl.mechanisms = PLAIN + +# Credentials +ckan.consumer.kafka.sasl.username = your_username +ckan.consumer.kafka.sasl.password = your_password + +### Tuning & Reliability +# Best practice for higher availability in librdkafka clients +ckan.consumer.kafka.session.timeout.ms = 45000 +ckan.consumer.kafka.auto.offset.reset = earliest diff --git a/ckanext/__init__.py b/ckanext/__init__.py new file mode 100644 index 0000000..3c17311 --- /dev/null +++ b/ckanext/__init__.py @@ -0,0 +1,7 @@ +# This file serves as a namespace package +try: + import pkg_resources + pkg_resources.declare_namespace(__name__) +except ImportError: + import pkgutil + __path__ = pkgutil.extend_path(__path__, __name__) diff --git a/ckanext/consumer/__init__.py b/ckanext/consumer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ckanext/consumer/consumer.py b/ckanext/consumer/consumer.py new file mode 100644 index 0000000..d953f5a --- /dev/null +++ b/ckanext/consumer/consumer.py @@ -0,0 +1,132 @@ +import logging +import sys +import json +from confluent_kafka import Consumer, KafkaError + +log = logging.getLogger(__name__) + + +def get_kafka_config(ckan_config): + """ + Extracts and validates Kafka configuration from CKAN .ini file. + Supports SASL/SSL authentication and client tuning. + """ + # 1. Base Required Configuration + bootstrap_servers = ckan_config.get('ckan.reactor.kafka.bootstrap.servers') + group_id = ckan_config.get('ckan.reactor.kafka.group_id') + + if not all([bootstrap_servers, group_id]): + log.error( + "Missing required config. Set 'ckan.reactor.kafka.bootstrap.servers' and 'ckan.reactor.kafka.group_id' in ckan.ini") + sys.exit(1) + + # Initialize config with defaults + conf = { + 'bootstrap.servers': bootstrap_servers, + 'group.id': group_id, + 'enable.auto.commit': True, + 'auto.offset.reset': ckan_config.get('ckan.reactor.kafka.auto.offset.reset', 'earliest'), + } + + # 2. Extended Configuration (Security & Tuning) + # Map CKAN .ini keys to librdkafka properties + config_mapping = { + # Identification + 'ckan.reactor.kafka.client.id': 'client.id', + + # Security Protocol (PLAINTEXT, SASL_SSL, etc.) + 'ckan.reactor.kafka.security.protocol': 'security.protocol', + + # SASL Auth + 'ckan.reactor.kafka.sasl.mechanisms': 'sasl.mechanisms', + 'ckan.reactor.kafka.sasl.username': 'sasl.username', + 'ckan.reactor.kafka.sasl.password': 'sasl.password', + + # Connection Tuning + 'ckan.reactor.kafka.session.timeout.ms': 'session.timeout.ms', + 'ckan.reactor.kafka.socket.timeout.ms': 'socket.timeout.ms', + } + + # 3. Apply optional settings if they exist in .ini + for ckan_key, kafka_key in config_mapping.items(): + value = ckan_config.get(ckan_key) + if value: + conf[kafka_key] = value + + return conf + + +def process_message(msg, handlers): + """ + Routes the message to the correct handler based on the topic. + Implements Resilient Error Handling strategy. + """ + topic = msg.topic() + msg_value = msg.value() + + handler = handlers.get(topic) + + if not handler: + # Avoid log spam if subscribing to patterns, but useful for debugging + # log.debug(f"No handler registered for topic '{topic}'. Skipping.") + return + + try: + if msg_value: + # Assumes JSON payload (CloudEvents) + data = json.loads(msg_value.decode('utf-8')) + else: + data = {} + + log.info(f"⚡ Processing event from topic: {topic}") + handler(data) + + except json.JSONDecodeError: + log.error(f"❌ JSON Decode Error in topic '{topic}'") + except Exception as e: + log.error(f"❌ Error executing handler for topic '{topic}': {e}", exc_info=True) + + +def run_consumer(ckan_config, topic_handlers): + """ + Main infinite loop for the Kafka consumer. + """ + conf = get_kafka_config(ckan_config) + + # Initialize Consumer + try: + consumer = Consumer(conf) + except Exception as e: + log.error(f"Failed to initialize Kafka Consumer: {e}") + return + + topics_to_subscribe = list(topic_handlers.keys()) + + if not topics_to_subscribe: + log.error("No topic handlers registered. Exiting.") + return + + try: + consumer.subscribe(topics_to_subscribe) + log.info(f"⚛️ Reactor Consumer started as client: {conf.get('client.id', 'unknown')}") + log.info(f"🎧 Listening on topics: {topics_to_subscribe}") + + while True: + # Poll with 1.0s timeout + msg = consumer.poll(1.0) + + if msg is None: + continue + if msg.error(): + if msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + log.error(f"Kafka Protocol Error: {msg.error()}") + continue + + process_message(msg, topic_handlers) + + except KeyboardInterrupt: + log.info("Reactor stopped by user.") + finally: + consumer.close() diff --git a/ckanext/consumer/interfaces.py b/ckanext/consumer/interfaces.py new file mode 100644 index 0000000..6bb5a5d --- /dev/null +++ b/ckanext/consumer/interfaces.py @@ -0,0 +1,26 @@ +from ckan.plugins.interfaces import Interface + + +class IConsumer(Interface): + """Interface for registering event handlers from other extensions. + + Any plugin implementing this interface can subscribe to Kafka topics + handled by the consumer. + """ + + def get_event_handlers(self): + """ + Returns a dictionary mapping topic names to handler functions. + + The handler function must accept a single argument: the data payload (dict). + + Example: + return { + 'com.opengov.users.create': self.handle_user_create, + 'com.opengov.datasets.update': self.handle_dataset_update + } + + Returns: + dict: { 'topic_name': callable_function } + """ + return {} diff --git a/ckanext/consumer/plugin.py b/ckanext/consumer/plugin.py new file mode 100644 index 0000000..11f0bd3 --- /dev/null +++ b/ckanext/consumer/plugin.py @@ -0,0 +1,55 @@ +import click +import ckan.plugins as plugins +import ckan.plugins.toolkit as toolkit + +from ckanext.consumer.interfaces import IConsumer +from ckanext.consumer.consumer import run_consumer + + +class ConsumerPlugin(plugins.SingletonPlugin): + """CKAN Consumer: Event-driven infrastructure plugin. + + 1. Registers the 'ckan consumer consume' CLI command. + 2. Collects event handlers from other plugins via IConsumer. + """ + + plugins.implements(plugins.IClick) + + def get_commands(self): + return [consumer] + + +@click.group() +def consumer(): + """Commands for the Consumer event system.""" + pass + + +@consumer.command() +def consume(): + """Starts the Kafka consumer worker process.""" + click.echo("Initializing Consumer environment...") + + topic_handlers = {} + + # --- Dynamic Handler Registration --- + # Iterate over all active plugins that implement IConsumer + for plugin in plugins.PluginImplementations(IConsumer): + try: + handlers = plugin.get_event_handlers() + if handlers: + click.echo(f" > Loaded handlers from extension '{plugin.name}': {list(handlers.keys())}") + topic_handlers.update(handlers) + except Exception as e: + click.echo(f" ! Error loading handlers from {plugin.name}: {e}") + + if not topic_handlers: + click.echo("⚠️ No handlers registered. The worker will start but will be idle.") + else: + click.echo(f"✅ Total topics monitored: {len(topic_handlers)}") + + # Load CKAN config + conf = toolkit.config + + # Start the consumer loop (blocking) + run_consumer(conf, topic_handlers) diff --git a/ckanext/tests/conftest.py b/ckanext/tests/conftest.py new file mode 100644 index 0000000..4d3ecc3 --- /dev/null +++ b/ckanext/tests/conftest.py @@ -0,0 +1,113 @@ +import sys +import types +from pathlib import Path + +import pytest + + +class DummyKafkaError: + _PARTITION_EOF = 123 + + +@pytest.fixture +def extension_root(): + return Path(__file__).resolve().parents[1] + + +@pytest.fixture(autouse=True) +def _ensure_extension_on_syspath(extension_root): + if str(extension_root) not in sys.path: + sys.path.insert(0, str(extension_root)) + + +@pytest.fixture +def reactor_consumer_module(monkeypatch): + """Import `ckanext.reactor.consumer` with confluent_kafka stubbed.""" + + dummy_kafka = types.SimpleNamespace(Consumer=object, KafkaError=DummyKafkaError) + monkeypatch.setitem(sys.modules, "confluent_kafka", dummy_kafka) + + import importlib + + return importlib.import_module("ckanext.reactor.consumer") + + +@pytest.fixture +def ckan_stub(monkeypatch): + """Provide minimal `ckan` module stubs for importing extension code.""" + + class _SingletonPlugin: + pass + + class _IClick: + pass + + def _implements(_iface): + return None + + class _Interface: + pass + + plugins_mod = types.SimpleNamespace( + SingletonPlugin=_SingletonPlugin, + IClick=_IClick, + implements=_implements, + PluginImplementations=lambda _iface: [], + ) + + plugins_interfaces_mod = types.ModuleType("ckan.plugins.interfaces") + plugins_interfaces_mod.Interface = _Interface + toolkit_mod = types.ModuleType("ckan.plugins.toolkit") + toolkit_mod.config = {} + + ckan_mod = types.ModuleType("ckan") + + plugins_pkg = types.ModuleType("ckan.plugins") + plugins_pkg.__dict__.update(plugins_mod.__dict__) + plugins_pkg.__path__ = [] + + ckan_mod.plugins = plugins_pkg + + monkeypatch.setitem(sys.modules, "ckan", ckan_mod) + monkeypatch.setitem(sys.modules, "ckan.plugins", plugins_pkg) + monkeypatch.setitem(sys.modules, "ckan.plugins.toolkit", toolkit_mod) + monkeypatch.setitem(sys.modules, "ckan.plugins.interfaces", plugins_interfaces_mod) + + # Also expose as attributes for `import ckan.plugins.toolkit as toolkit`. + plugins_pkg.toolkit = toolkit_mod + plugins_pkg.interfaces = plugins_interfaces_mod + + return types.SimpleNamespace(plugins=plugins_mod, toolkit=toolkit_mod) + + +@pytest.fixture +def reactor_plugin_module(monkeypatch, ckan_stub): + """Import `ckanext.reactor.plugin` with CKAN + click + confluent stubbed.""" + + # Stub `confluent_kafka` so `consumer.py` import works. + dummy_kafka = types.SimpleNamespace(Consumer=object, KafkaError=DummyKafkaError) + monkeypatch.setitem(sys.modules, "confluent_kafka", dummy_kafka) + + # Stub `click` so we can import plugin.py. + def _group(): + def decorator(func): + class _Group: + def __init__(self, callback): + self.callback = callback + + def command(self): + def command_decorator(command_callback): + return types.SimpleNamespace(callback=command_callback) + + return command_decorator + + return _Group(func) + + return decorator + + click_mod = types.SimpleNamespace(group=_group, echo=lambda _msg: None) + monkeypatch.setitem(sys.modules, "click", click_mod) + + import importlib + + return importlib.import_module("ckanext.reactor.plugin") diff --git a/ckanext/tests/test_consumer_get_kafka_config.py b/ckanext/tests/test_consumer_get_kafka_config.py new file mode 100644 index 0000000..a703ced --- /dev/null +++ b/ckanext/tests/test_consumer_get_kafka_config.py @@ -0,0 +1,41 @@ +import pytest + + +def test_get_kafka_config_requires_bootstrap_and_group_id(reactor_consumer_module, caplog): + consumer = reactor_consumer_module + + with pytest.raises(SystemExit) as excinfo: + consumer.get_kafka_config({}) + + assert excinfo.value.code == 1 + assert "Missing required config" in caplog.text + + +def test_get_kafka_config_uses_default_offset_reset(reactor_consumer_module): + consumer = reactor_consumer_module + + cfg = { + "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", + "ckan.reactor.kafka.group_id": "group", + } + + kafka_cfg = consumer.get_kafka_config(cfg) + + assert kafka_cfg["bootstrap.servers"] == "localhost:9092" + assert kafka_cfg["group.id"] == "group" + assert kafka_cfg["auto.offset.reset"] == "earliest" + assert kafka_cfg["enable.auto.commit"] is True + + +def test_get_kafka_config_allows_override_offset_reset(reactor_consumer_module): + consumer = reactor_consumer_module + + cfg = { + "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", + "ckan.reactor.kafka.group_id": "group", + "ckan.reactor.kafka.auto.offset.reset": "latest", + } + + kafka_cfg = consumer.get_kafka_config(cfg) + + assert kafka_cfg["auto.offset.reset"] == "latest" diff --git a/ckanext/tests/test_consumer_process_message.py b/ckanext/tests/test_consumer_process_message.py new file mode 100644 index 0000000..4c23950 --- /dev/null +++ b/ckanext/tests/test_consumer_process_message.py @@ -0,0 +1,79 @@ +import json + + +class DummyMsg: + def __init__(self, topic, value): + self._topic = topic + self._value = value + + def topic(self): + return self._topic + + def value(self): + return self._value + + +def test_process_message_skips_when_no_handler(reactor_consumer_module, caplog): + consumer = reactor_consumer_module + + msg = DummyMsg("topic-a", b"{}") + consumer.process_message(msg, handlers={}) + + assert "no handler is registered" in caplog.text + + +def test_process_message_calls_handler_with_decoded_json(reactor_consumer_module): + consumer = reactor_consumer_module + + called = {"data": None} + + def handler(data): + called["data"] = data + + payload = {"hello": "world"} + msg = DummyMsg("topic-a", json.dumps(payload).encode("utf-8")) + + consumer.process_message(msg, handlers={"topic-a": handler}) + + assert called["data"] == payload + + +def test_process_message_passes_empty_dict_for_null_value(reactor_consumer_module): + consumer = reactor_consumer_module + + called = {"data": None} + + def handler(data): + called["data"] = data + + msg = DummyMsg("topic-a", None) + + consumer.process_message(msg, handlers={"topic-a": handler}) + + assert called["data"] == {} + + +def test_process_message_logs_json_decode_error(reactor_consumer_module, caplog): + consumer = reactor_consumer_module + + def handler(_data): + raise AssertionError("handler should not be called") + + msg = DummyMsg("topic-a", b"not-json") + + consumer.process_message(msg, handlers={"topic-a": handler}) + + assert "JSON Decode Error" in caplog.text + + +def test_process_message_logs_handler_exception(reactor_consumer_module, caplog): + consumer = reactor_consumer_module + + def handler(_data): + raise ValueError("boom") + + msg = DummyMsg("topic-a", b"{}") + + consumer.process_message(msg, handlers={"topic-a": handler}) + + assert "Error executing handler" in caplog.text diff --git a/ckanext/tests/test_consumer_run_consumer.py b/ckanext/tests/test_consumer_run_consumer.py new file mode 100644 index 0000000..245c89d --- /dev/null +++ b/ckanext/tests/test_consumer_run_consumer.py @@ -0,0 +1,91 @@ +class DummyKafkaError: + _PARTITION_EOF = 123 + + +class DummyErr: + def __init__(self, code): + self._code = code + + def code(self): + return self._code + + def __str__(self): + return f"err({self._code})" + + +class DummyMsg: + def __init__(self, value=None, err=None): + self._value = value + self._err = err + + def error(self): + return self._err + + def value(self): + return self._value + + +class DummyConsumer: + def __init__(self, conf): + self.conf = conf + self.subscribed = None + self.closed = False + self.poll_calls = 0 + + def subscribe(self, topics): + self.subscribed = topics + + def poll(self, _timeout): + self.poll_calls += 1 + if self.poll_calls == 1: + return None + if self.poll_calls == 2: + return DummyMsg(err=DummyErr(DummyKafkaError._PARTITION_EOF)) + if self.poll_calls == 3: + return DummyMsg(err=DummyErr(999)) + raise KeyboardInterrupt() + + def close(self): + self.closed = True + + +def test_run_consumer_exits_when_no_topics(reactor_consumer_module, monkeypatch, caplog): + consumer = reactor_consumer_module + + # Avoid instantiating the real Consumer. + monkeypatch.setattr(consumer, "Consumer", DummyConsumer) + + consumer.run_consumer( + { + "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", + "ckan.reactor.kafka.group_id": "group", + }, + {}, + ) + + assert "nothing to listen" in caplog.text + + +def test_run_consumer_subscribes_and_closes(reactor_consumer_module, monkeypatch): + consumer = reactor_consumer_module + + monkeypatch.setattr(consumer, "KafkaError", DummyKafkaError) + monkeypatch.setattr(consumer, "Consumer", DummyConsumer) + + called = {"count": 0} + + def fake_process_message(_msg, _handlers): + called["count"] += 1 + + monkeypatch.setattr(consumer, "process_message", fake_process_message) + + ckan_cfg = { + "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", + "ckan.reactor.kafka.group_id": "group", + } + + handlers = {"topic-a": lambda _data: None} + + consumer.run_consumer(ckan_cfg, handlers) + + assert called["count"] == 0 diff --git a/ckanext/tests/test_interfaces.py b/ckanext/tests/test_interfaces.py new file mode 100644 index 0000000..be4d974 --- /dev/null +++ b/ckanext/tests/test_interfaces.py @@ -0,0 +1,5 @@ +def test_iconsumer_default_handlers_is_empty_dict(ckan_stub): + from ckanext.consumer.interfaces import IConsumer + + iface = IConsumer() + assert iface.get_event_handlers() == {} diff --git a/ckanext/tests/test_plugin.py b/ckanext/tests/test_plugin.py new file mode 100644 index 0000000..0fbc1d5 --- /dev/null +++ b/ckanext/tests/test_plugin.py @@ -0,0 +1,73 @@ +def test_plugin_get_commands_returns_reactor_group(reactor_plugin_module): + plugin_mod = reactor_plugin_module + + plugin = plugin_mod.ReactorPlugin() + commands = plugin.get_commands() + + assert commands == [plugin_mod.reactor] + + +def test_consume_warns_when_no_handlers(monkeypatch, reactor_plugin_module): + plugin_mod = reactor_plugin_module + + echoed = [] + monkeypatch.setattr(plugin_mod.click, "echo", lambda msg: echoed.append(msg)) + + # No plugins implementing IReactor. + monkeypatch.setattr(plugin_mod.plugins, "PluginImplementations", lambda _iface: []) + + # Avoid blocking consumer loop. + called = {"conf": None, "handlers": None} + + def fake_run_consumer(conf, handlers): + called["conf"] = conf + called["handlers"] = handlers + + monkeypatch.setattr(plugin_mod, "run_consumer", fake_run_consumer) + monkeypatch.setattr(plugin_mod.toolkit, "config", {"x": "y"}) + + plugin_mod.consume.callback() + + assert any("No handlers registered" in m for m in echoed) + assert called["conf"] == {"x": "y"} + assert called["handlers"] == {} + + +def test_consume_loads_handlers_and_continues_on_error(monkeypatch, reactor_plugin_module): + plugin_mod = reactor_plugin_module + + echoed = [] + monkeypatch.setattr(plugin_mod.click, "echo", lambda msg: echoed.append(msg)) + + class GoodPlugin: + name = "good_ext" + + def get_event_handlers(self): + return {"topic-a": lambda _data: None} + + class BadPlugin: + name = "bad_ext" + + def get_event_handlers(self): + raise RuntimeError("boom") + + monkeypatch.setattr( + plugin_mod.plugins, + "PluginImplementations", + lambda _iface: [GoodPlugin(), BadPlugin()], + ) + + called = {"handlers": None} + + def fake_run_consumer(_conf, handlers): + called["handlers"] = handlers + + monkeypatch.setattr(plugin_mod, "run_consumer", fake_run_consumer) + monkeypatch.setattr(plugin_mod.toolkit, "config", {}) + + plugin_mod.consume.callback() + + assert called["handlers"] == {"topic-a": called["handlers"]["topic-a"]} + assert any("Loaded handlers" in m for m in echoed) + assert any("Error loading handlers" in m for m in echoed) + assert any("Total topics monitored" in m for m in echoed) diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..d5ffc35 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,3 @@ +pytest-ckan +pytest-cov +factory-boy diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..f607c7b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +python_files = test_*.py +testpaths = tests diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..42edc69 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +confluent-kafka>=2.0.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..02ed806 --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +from setuptools import setup, find_packages + +setup( + name='ckanext-consumer', + version='0.1.0', + description='Event-driven architecture extension for CKAN (Kafka Consumer)', + long_description='Listens to Kafka topics and dispatches events to registered handlers in other CKAN extensions.', + classifiers=[ + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + ], + author='Peter Vorman', + author_email='pvorman@opengov.com', + url='https://github.com/OpenGov-OpenData/ckanext-consumer', + license='GNU AFFERO GENERAL PUBLIC LICENSE', + packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + namespace_packages=['ckanext'], + include_package_data=True, + zip_safe=False, + install_requires=[], + entry_points=''' + [ckan.plugins] + consumer=ckanext.consumer.plugin:ConsumerPlugin + + [babel.extractors] + ckan = ckan.lib.extract:extract_ckan + ''', +) From a241723706d5a6c9c996bdbd1d1bb6223702e81f Mon Sep 17 00:00:00 2001 From: Peter Vorman Date: Tue, 10 Feb 2026 17:01:42 +0200 Subject: [PATCH 2/2] ci --- .github/CODEOWNERS | 7 ++ .github/workflows/test.yml | 67 +++++++++++++++++ ckanext/consumer/consumer.py | 30 ++++---- ckanext/consumer/plugin.py | 1 - ckanext/tests/conftest.py | 12 ++-- .../tests/test_consumer_get_kafka_config.py | 22 +++--- .../tests/test_consumer_process_message.py | 25 +++---- ckanext/tests/test_consumer_run_consumer.py | 18 ++--- ckanext/tests/test_plugin.py | 18 ++--- test.ini | 72 +++++++++++++++++++ 10 files changed, 205 insertions(+), 67 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/test.yml create mode 100644 test.ini diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..00bd2b0 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# RESOURCES +# - https://git-scm.com/docs/gitignore#_pattern_format +# - https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners +# ---------------------------------------------------------------- +### + +* @OpenGov-OpenData/CKAN diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..aa2fc42 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,67 @@ +name: Tests +on: [pull_request] +env: + CODE_COVERAGE_THRESHOLD_REQUIRED: 90 +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan + - name: Run flake8 + run: flake8 . --count --max-line-length=127 --statistics --exclude ckan + + test: + needs: lint + strategy: + matrix: + include: + - ckan-version: "2.9" + ckan-image: "ckan/ckan-dev:2.9-py3.9" + fail-fast: false + + name: CKAN ${{ matrix.ckan-version }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.ckan-image }} + options: --user root + services: + solr: + image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9 + postgres: + image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:3 + env: + CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test + CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test + CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test + CKAN_SOLR_URL: http://solr:8983/solr/ckan + CKAN_REDIS_URL: redis://redis:6379/1 + + steps: + - uses: actions/checkout@v5 + - name: Install requirements (common) + run: | + pip install -r requirements.txt + pip install -r dev-requirements.txt + pip install -e . + # Replace default path to CKAN core config file with the one on the container + sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Setup extension (CKAN >= 2.9) + run: | + ckan -c test.ini db init + - name: Run tests + run: | + pytest --cov=ckanext.consumer --ckan-ini=test.ini --disable-warnings --cov-fail-under=${CODE_COVERAGE_THRESHOLD_REQUIRED} ckanext/tests/ diff --git a/ckanext/consumer/consumer.py b/ckanext/consumer/consumer.py index d953f5a..62a0d0d 100644 --- a/ckanext/consumer/consumer.py +++ b/ckanext/consumer/consumer.py @@ -12,12 +12,14 @@ def get_kafka_config(ckan_config): Supports SASL/SSL authentication and client tuning. """ # 1. Base Required Configuration - bootstrap_servers = ckan_config.get('ckan.reactor.kafka.bootstrap.servers') - group_id = ckan_config.get('ckan.reactor.kafka.group_id') + bootstrap_servers = ckan_config.get('ckan.consumer.kafka.bootstrap.servers') + group_id = ckan_config.get('ckan.consumer.kafka.group_id') if not all([bootstrap_servers, group_id]): log.error( - "Missing required config. Set 'ckan.reactor.kafka.bootstrap.servers' and 'ckan.reactor.kafka.group_id' in ckan.ini") + "Missing required config. Set 'ckan.consumer.kafka.bootstrap.servers' and " + "'ckan.consumer.kafka.group_id' in ckan.ini" + ) sys.exit(1) # Initialize config with defaults @@ -25,26 +27,26 @@ def get_kafka_config(ckan_config): 'bootstrap.servers': bootstrap_servers, 'group.id': group_id, 'enable.auto.commit': True, - 'auto.offset.reset': ckan_config.get('ckan.reactor.kafka.auto.offset.reset', 'earliest'), + 'auto.offset.reset': ckan_config.get('ckan.consumer.kafka.auto.offset.reset', 'earliest'), } # 2. Extended Configuration (Security & Tuning) # Map CKAN .ini keys to librdkafka properties config_mapping = { # Identification - 'ckan.reactor.kafka.client.id': 'client.id', + 'ckan.consumer.kafka.client.id': 'client.id', # Security Protocol (PLAINTEXT, SASL_SSL, etc.) - 'ckan.reactor.kafka.security.protocol': 'security.protocol', + 'ckan.consumer.kafka.security.protocol': 'security.protocol', # SASL Auth - 'ckan.reactor.kafka.sasl.mechanisms': 'sasl.mechanisms', - 'ckan.reactor.kafka.sasl.username': 'sasl.username', - 'ckan.reactor.kafka.sasl.password': 'sasl.password', + 'ckan.consumer.kafka.sasl.mechanisms': 'sasl.mechanisms', + 'ckan.consumer.kafka.sasl.username': 'sasl.username', + 'ckan.consumer.kafka.sasl.password': 'sasl.password', # Connection Tuning - 'ckan.reactor.kafka.session.timeout.ms': 'session.timeout.ms', - 'ckan.reactor.kafka.socket.timeout.ms': 'socket.timeout.ms', + 'ckan.consumer.kafka.session.timeout.ms': 'session.timeout.ms', + 'ckan.consumer.kafka.socket.timeout.ms': 'socket.timeout.ms', } # 3. Apply optional settings if they exist in .ini @@ -68,7 +70,7 @@ def process_message(msg, handlers): if not handler: # Avoid log spam if subscribing to patterns, but useful for debugging - # log.debug(f"No handler registered for topic '{topic}'. Skipping.") + log.debug(f"No handler registered for topic '{topic}'. Skipping.") return try: @@ -108,7 +110,7 @@ def run_consumer(ckan_config, topic_handlers): try: consumer.subscribe(topics_to_subscribe) - log.info(f"⚛️ Reactor Consumer started as client: {conf.get('client.id', 'unknown')}") + log.info(f"⚛️ Consumer started as client: {conf.get('client.id', 'unknown')}") log.info(f"🎧 Listening on topics: {topics_to_subscribe}") while True: @@ -127,6 +129,6 @@ def run_consumer(ckan_config, topic_handlers): process_message(msg, topic_handlers) except KeyboardInterrupt: - log.info("Reactor stopped by user.") + log.info("Consumer stopped by user.") finally: consumer.close() diff --git a/ckanext/consumer/plugin.py b/ckanext/consumer/plugin.py index 11f0bd3..1e1103e 100644 --- a/ckanext/consumer/plugin.py +++ b/ckanext/consumer/plugin.py @@ -22,7 +22,6 @@ def get_commands(self): @click.group() def consumer(): """Commands for the Consumer event system.""" - pass @consumer.command() diff --git a/ckanext/tests/conftest.py b/ckanext/tests/conftest.py index 4d3ecc3..748159a 100644 --- a/ckanext/tests/conftest.py +++ b/ckanext/tests/conftest.py @@ -21,15 +21,15 @@ def _ensure_extension_on_syspath(extension_root): @pytest.fixture -def reactor_consumer_module(monkeypatch): - """Import `ckanext.reactor.consumer` with confluent_kafka stubbed.""" +def consumer_module(monkeypatch): + """Import `ckanext.consumer.consumer` with confluent_kafka stubbed.""" dummy_kafka = types.SimpleNamespace(Consumer=object, KafkaError=DummyKafkaError) monkeypatch.setitem(sys.modules, "confluent_kafka", dummy_kafka) import importlib - return importlib.import_module("ckanext.reactor.consumer") + return importlib.import_module("ckanext.consumer.consumer") @pytest.fixture @@ -81,8 +81,8 @@ class _Interface: @pytest.fixture -def reactor_plugin_module(monkeypatch, ckan_stub): - """Import `ckanext.reactor.plugin` with CKAN + click + confluent stubbed.""" +def consumer_plugin_module(monkeypatch, ckan_stub): + """Import `ckanext.consumer.plugin` with CKAN + click + confluent stubbed.""" # Stub `confluent_kafka` so `consumer.py` import works. dummy_kafka = types.SimpleNamespace(Consumer=object, KafkaError=DummyKafkaError) @@ -110,4 +110,4 @@ def command_decorator(command_callback): import importlib - return importlib.import_module("ckanext.reactor.plugin") + return importlib.import_module("ckanext.consumer.plugin") diff --git a/ckanext/tests/test_consumer_get_kafka_config.py b/ckanext/tests/test_consumer_get_kafka_config.py index a703ced..407dba3 100644 --- a/ckanext/tests/test_consumer_get_kafka_config.py +++ b/ckanext/tests/test_consumer_get_kafka_config.py @@ -1,8 +1,8 @@ import pytest -def test_get_kafka_config_requires_bootstrap_and_group_id(reactor_consumer_module, caplog): - consumer = reactor_consumer_module +def test_get_kafka_config_requires_bootstrap_and_group_id(consumer_module, caplog): + consumer = consumer_module with pytest.raises(SystemExit) as excinfo: consumer.get_kafka_config({}) @@ -11,12 +11,12 @@ def test_get_kafka_config_requires_bootstrap_and_group_id(reactor_consumer_modul assert "Missing required config" in caplog.text -def test_get_kafka_config_uses_default_offset_reset(reactor_consumer_module): - consumer = reactor_consumer_module +def test_get_kafka_config_uses_default_offset_reset(consumer_module): + consumer = consumer_module cfg = { - "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", - "ckan.reactor.kafka.group_id": "group", + "ckan.consumer.kafka.bootstrap.servers": "localhost:9092", + "ckan.consumer.kafka.group_id": "group", } kafka_cfg = consumer.get_kafka_config(cfg) @@ -27,13 +27,13 @@ def test_get_kafka_config_uses_default_offset_reset(reactor_consumer_module): assert kafka_cfg["enable.auto.commit"] is True -def test_get_kafka_config_allows_override_offset_reset(reactor_consumer_module): - consumer = reactor_consumer_module +def test_get_kafka_config_allows_override_offset_reset(consumer_module): + consumer = consumer_module cfg = { - "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", - "ckan.reactor.kafka.group_id": "group", - "ckan.reactor.kafka.auto.offset.reset": "latest", + "ckan.consumer.kafka.bootstrap.servers": "localhost:9092", + "ckan.consumer.kafka.group_id": "group", + "ckan.consumer.kafka.auto.offset.reset": "latest", } kafka_cfg = consumer.get_kafka_config(cfg) diff --git a/ckanext/tests/test_consumer_process_message.py b/ckanext/tests/test_consumer_process_message.py index 4c23950..9c9ccc5 100644 --- a/ckanext/tests/test_consumer_process_message.py +++ b/ckanext/tests/test_consumer_process_message.py @@ -13,17 +13,8 @@ def value(self): return self._value -def test_process_message_skips_when_no_handler(reactor_consumer_module, caplog): - consumer = reactor_consumer_module - - msg = DummyMsg("topic-a", b"{}") - consumer.process_message(msg, handlers={}) - - assert "no handler is registered" in caplog.text - - -def test_process_message_calls_handler_with_decoded_json(reactor_consumer_module): - consumer = reactor_consumer_module +def test_process_message_calls_handler_with_decoded_json(consumer_module): + consumer = consumer_module called = {"data": None} @@ -38,8 +29,8 @@ def handler(data): assert called["data"] == payload -def test_process_message_passes_empty_dict_for_null_value(reactor_consumer_module): - consumer = reactor_consumer_module +def test_process_message_passes_empty_dict_for_null_value(consumer_module): + consumer = consumer_module called = {"data": None} @@ -53,8 +44,8 @@ def handler(data): assert called["data"] == {} -def test_process_message_logs_json_decode_error(reactor_consumer_module, caplog): - consumer = reactor_consumer_module +def test_process_message_logs_json_decode_error(consumer_module, caplog): + consumer = consumer_module def handler(_data): raise AssertionError("handler should not be called") @@ -66,8 +57,8 @@ def handler(_data): assert "JSON Decode Error" in caplog.text -def test_process_message_logs_handler_exception(reactor_consumer_module, caplog): - consumer = reactor_consumer_module +def test_process_message_logs_handler_exception(consumer_module, caplog): + consumer = consumer_module def handler(_data): raise ValueError("boom") diff --git a/ckanext/tests/test_consumer_run_consumer.py b/ckanext/tests/test_consumer_run_consumer.py index 245c89d..f76618a 100644 --- a/ckanext/tests/test_consumer_run_consumer.py +++ b/ckanext/tests/test_consumer_run_consumer.py @@ -49,25 +49,25 @@ def close(self): self.closed = True -def test_run_consumer_exits_when_no_topics(reactor_consumer_module, monkeypatch, caplog): - consumer = reactor_consumer_module +def test_run_consumer_exits_when_no_topics(consumer_module, monkeypatch, caplog): + consumer = consumer_module # Avoid instantiating the real Consumer. monkeypatch.setattr(consumer, "Consumer", DummyConsumer) consumer.run_consumer( { - "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", - "ckan.reactor.kafka.group_id": "group", + "ckan.consumer.kafka.bootstrap.servers": "localhost:9092", + "ckan.consumer.kafka.group_id": "group", }, {}, ) - assert "nothing to listen" in caplog.text + assert "No topic handlers registered" in caplog.text -def test_run_consumer_subscribes_and_closes(reactor_consumer_module, monkeypatch): - consumer = reactor_consumer_module +def test_run_consumer_subscribes_and_closes(consumer_module, monkeypatch): + consumer = consumer_module monkeypatch.setattr(consumer, "KafkaError", DummyKafkaError) monkeypatch.setattr(consumer, "Consumer", DummyConsumer) @@ -80,8 +80,8 @@ def fake_process_message(_msg, _handlers): monkeypatch.setattr(consumer, "process_message", fake_process_message) ckan_cfg = { - "ckan.reactor.kafka.bootstrap.servers": "localhost:9092", - "ckan.reactor.kafka.group_id": "group", + "ckan.consumer.kafka.bootstrap.servers": "localhost:9092", + "ckan.consumer.kafka.group_id": "group", } handlers = {"topic-a": lambda _data: None} diff --git a/ckanext/tests/test_plugin.py b/ckanext/tests/test_plugin.py index 0fbc1d5..8fb403b 100644 --- a/ckanext/tests/test_plugin.py +++ b/ckanext/tests/test_plugin.py @@ -1,19 +1,19 @@ -def test_plugin_get_commands_returns_reactor_group(reactor_plugin_module): - plugin_mod = reactor_plugin_module +def test_plugin_get_commands_returns_consumer_group(consumer_plugin_module): + plugin_mod = consumer_plugin_module - plugin = plugin_mod.ReactorPlugin() + plugin = plugin_mod.ConsumerPlugin() commands = plugin.get_commands() - assert commands == [plugin_mod.reactor] + assert commands == [plugin_mod.consumer] -def test_consume_warns_when_no_handlers(monkeypatch, reactor_plugin_module): - plugin_mod = reactor_plugin_module +def test_consume_warns_when_no_handlers(monkeypatch, consumer_plugin_module): + plugin_mod = consumer_plugin_module echoed = [] monkeypatch.setattr(plugin_mod.click, "echo", lambda msg: echoed.append(msg)) - # No plugins implementing IReactor. + # No plugins implementing IConsumer. monkeypatch.setattr(plugin_mod.plugins, "PluginImplementations", lambda _iface: []) # Avoid blocking consumer loop. @@ -33,8 +33,8 @@ def fake_run_consumer(conf, handlers): assert called["handlers"] == {} -def test_consume_loads_handlers_and_continues_on_error(monkeypatch, reactor_plugin_module): - plugin_mod = reactor_plugin_module +def test_consume_loads_handlers_and_continues_on_error(monkeypatch, consumer_plugin_module): + plugin_mod = consumer_plugin_module echoed = [] monkeypatch.setattr(plugin_mod.click, "echo", lambda msg: echoed.append(msg)) diff --git a/test.ini b/test.ini new file mode 100644 index 0000000..852f67c --- /dev/null +++ b/test.ini @@ -0,0 +1,72 @@ +[DEFAULT] +debug = false +# Uncomment and replace with the address which should receive any error reports +#email_to = you@yourdomain.com +smtp_server = localhost +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + + +[app:main] +use = config:../ckan/test-core.ini +# Here we hard-code the database and a flag to make default tests +# run fast. +ckan.plugins = consumer + +### Basic Configuration +ckan.consumer.kafka.bootstrap.servers = some.confluent.cloud:9092 +ckan.consumer.kafka.group_id = consumer-group +ckan.consumer.kafka.client.id = ccloud-python + +### Security & Authentication (SASL/SSL) +# Protocol: PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL +ckan.consumer.kafka.security.protocol = SASL_SSL + +# Mechanism: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, GSSAPI (Kerberos) +ckan.consumer.kafka.sasl.mechanisms = PLAIN + +# Credentials +ckan.consumer.kafka.sasl.username = +ckan.consumer.kafka.sasl.password = + +### Tuning & Reliability +# Best practice for higher availability in librdkafka clients +ckan.consumer.kafka.session.timeout.ms = 45000 +ckan.consumer.kafka.auto.offset.reset = earliest + +# Logging configuration +[loggers] +keys = root, ckan, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_ckan] +qualname = ckan +handlers = +level = INFO + +[logger_sqlalchemy] +handlers = +qualname = sqlalchemy.engine +level = ERROR + +[handler_console] +class = StreamHandler +args = (sys.stdout,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s