diff --git a/.gitignore b/.gitignore index 28fc69c..c42ff0b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ www/assets/ mediamtx/ onnxruntime/ +capture/ + +python/ diff --git a/Cargo.toml b/Cargo.toml index 542e828..4ecfdf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "bevy_light_field" description = "rust bevy light field array tooling" -version = "0.5.0" +version = "0.7.0" edition = "2021" authors = ["mosure "] license = "MIT" @@ -28,21 +28,28 @@ default-run = "viewer" [features] default = [ "person_matting", + "pipeline", ] person_matting = ["bevy_ort", "ort", "ndarray"] +pipeline = ["image", "imageproc", "rayon"] +yolo = ["bevy_ort", "ort", "ndarray"] [dependencies] anyhow = "1.0" async-compat = "0.2" bevy_args = "1.3" -bevy_ort = { version = "0.6", optional = true } +bevy_ort = { version = "0.8", optional = true, features = ["yolo_v8"] } bytes = "1.5" clap = { version = "4.4", features = ["derive"] } futures = "0.3" +image = { version = "0.24", optional = true } # update /w `bevy` crate +imageproc = { version = "0.23.0", optional = true } # update /w `image` crate ndarray = { version = "0.15", optional = true } openh264 = "0.5" +png = "0.17.13" +rayon = { version = "1.8", optional = true } serde = "1.0" serde_json = "1.0" serde_qs = "0.12" @@ -76,6 +83,10 @@ features = [ ] +[dev-dependencies] +approx = "0.5" + + [profile.dev.package."*"] opt-level = 3 diff --git a/LICENSE-GPL-3.0 b/LICENSE-GPL-3.0 new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE-GPL-3.0 @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/LICENSE b/LICENSE-MIT similarity index 100% rename from LICENSE rename to LICENSE-MIT diff --git a/README.md b/README.md index ef752ff..466de4e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ # bevy_light_field πŸ’‘πŸŒΎπŸ“· [![test](https://github.com/mosure/bevy_light_field/workflows/test/badge.svg)](https://github.com/Mosure/bevy_light_field/actions?query=workflow%3Atest) -[![GitHub License](https://img.shields.io/github/license/mosure/bevy_light_field)](https://raw.githubusercontent.com/mosure/bevy_light_field/main/LICENSE) -[![GitHub Last Commit](https://img.shields.io/github/last-commit/mosure/bevy_light_field)](https://github.com/mosure/bevy_light_field) -[![GitHub Releases](https://img.shields.io/github/v/release/mosure/bevy_light_field?include_prereleases&sort=semver)](https://github.com/mosure/bevy_light_field/releases) -[![GitHub Issues](https://img.shields.io/github/issues/mosure/bevy_light_field)](https://github.com/mosure/bevy_light_field/issues) -[![Average time to resolve an issue](https://isitmaintained.com/badge/resolution/mosure/bevy_light_field.svg)](http://isitmaintained.com/project/mosure/bevy_light_field) +[![GitHub License](https://img.shields.io/badge/license-MIT%2FGPL%E2%80%933.0-blue.svg)](https://github.com/mosure/bevy_ort#license) [![crates.io](https://img.shields.io/crates/v/bevy_light_field.svg)](https://crates.io/crates/bevy_light_field) rust bevy light field camera array tooling @@ -17,9 +13,9 @@ rust bevy light field camera array tooling - [X] person segmentation post-process (batch across streams) - [X] async segmentation model inference - [X] foreground extraction post-process and visualization mode +- [X] recording session viewer - [ ] camera array calibration (extrinsics, intrinsics, color) - [ ] camera position visualization -- [ ] playback nersemble recordings with annotations - [ ] 3d reconstruction dataset preparation - [ ] real-time 3d reconstruction viewer @@ -46,90 +42,49 @@ the viewer opens a window and displays the light field camera array, with post-p ## library usage ```rust -use bevy::{ - prelude::*, - render::render_resource::{ - Extent3d, - TextureDescriptor, - TextureDimension, - TextureFormat, - TextureUsages, - }, -}; +use bevy::prelude::*; -use bevy_light_field::stream::{ - RtspStreamDescriptor, - RtspStreamPlugin, - StreamId, +use bevy_light_field::{ + LightFieldPlugin, + stream::RtspStreamHandle, }; - -const RTSP_URIS: [&str; 1] = [ - "rtsp://localhost:554/lizard", -]; - - fn main() { App::new() .add_plugins(( DefaultPlugins, - RtspStreamPlugin, + LightFieldPlugin { + stream_config: "assets/streams.json", + }, )) - .add_systems(Startup, create_streams) + .add_systems(Startup, setup_ui_gridview) .run(); } - -fn create_streams( +fn setup_ui_gridview( mut commands: Commands, - mut images: ResMut>, + input_streams: Query<( + Entity, + &RtspStreamHandle, + )>, ) { - RTSP_URIS.iter() - .enumerate() - .for_each(|(index, &url)| { - let entity = commands.spawn_empty().id(); - - let size = Extent3d { - width: 640, - height: 360, - ..default() - }; - - let mut image = Image { - texture_descriptor: TextureDescriptor { - label: Some(url), - size, - dimension: TextureDimension::D2, - format: TextureFormat::Rgba8UnormSrgb, - mip_level_count: 1, - sample_count: 1, - usage: TextureUsages::COPY_DST - | TextureUsages::TEXTURE_BINDING - | TextureUsages::RENDER_ATTACHMENT, - view_formats: &[TextureFormat::Rgba8UnormSrgb], - }, - ..default() - }; - image.resize(size); - - let image = images.add(image); - commands.spawn(SpriteBundle { - sprite: Sprite { - custom_size: Some(Vec2::new(640.0, 360.0)), - ..default() - }, - texture: image.clone(), - ..default() - }); - - let rtsp_stream = RtspStreamDescriptor::new( - url.to_string(), - StreamId(index), - image, - ); - - commands.entity(entity).insert(rtsp_stream); - }); + let stream = input_streams.single().unwrap(); + + commands.spawn(ImageBundle { + style: Style { + width: Val::Percent(100.0), + height: Val::Percent(100.0), + ..default() + }, + image: UiImage::new(stream.image.clone()), + ..default() + }); + + commands.spawn(( + Camera2dBundle { + ..default() + }, + )); } ``` @@ -143,17 +98,6 @@ view the [onshape model](https://cad.onshape.com/documents/20d4b522e97cda88fb785 ![Alt text](docs/light_field_camera_onshape_transparent.webp) -## setup rtsp streaming server - -it is useful to test the light field viewer with emulated camera streams - -### obs studio - -- install https://obsproject.com/ -- install rtsp plugin https://github.com/iamscottxu/obs-rtspserver/releases -- tools > rtsp server > start server - - ## compatible bevy versions | `bevy_light_field` | `bevy` | @@ -167,4 +111,16 @@ it is useful to test the light field viewer with emulated camera streams - [modnet](https://github.com/ZHKKKe/MODNet) - [nersemble](https://github.com/tobias-kirschstein/nersemble) - [paddle_seg_matting](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.9/Matting/docs/quick_start_en.md) +- [pose diffusion](https://github.com/facebookresearch/PoseDiffusion) - [ray diffusion](https://github.com/jasonyzhang/RayDiffusion) + + +## license + +This software is dual-licensed under the MIT License and the GNU General Public License version 3 (GPL-3.0). + +You may choose to use this software under the terms of the MIT License OR the GNU General Public License version 3 (GPL-3.0), except as stipulated below: + +The use of the `yolo_v8` feature within this software is specifically governed by the GNU General Public License version 3 (GPL-3.0). By using the `yolo_v8` feature, you agree to comply with the terms and conditions of the GPL-3.0. + +For more details on the licenses, please refer to the LICENSE.MIT and LICENSE.GPL-3.0 files included with this software. diff --git a/assets/modnet_photographic_portrait_matting.onnx b/assets/models/modnet_photographic_portrait_matting.onnx similarity index 100% rename from assets/modnet_photographic_portrait_matting.onnx rename to assets/models/modnet_photographic_portrait_matting.onnx diff --git a/assets/models/yolov8n.onnx b/assets/models/yolov8n.onnx new file mode 100644 index 0000000..cfb621b Binary files /dev/null and b/assets/models/yolov8n.onnx differ diff --git a/assets/streams.json b/assets/streams.json index 6b8838a..58599cf 100644 --- a/assets/streams.json +++ b/assets/streams.json @@ -1,10 +1,20 @@ [ - "rtsp://192.168.1.23/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.24/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.25/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.26/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.27/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.28/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.29/user=admin&password=admin123&channel=1&stream=0.sdp?", - "rtsp://192.168.1.30/user=admin&password=admin123&channel=1&stream=0.sdp?" + { "uri": "rtsp://192.168.1.21/stream/main", "transport": "Udp", "visible": true }, + { "uri": "rtsp://192.168.1.22/stream/main", "transport": "Udp", "visible": true, "person_detection": true }, + + { "uri": "rtsp://192.168.1.23/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 45.0, "visible": true }, + { "uri": "rtsp://192.168.1.24/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 55.0 }, + { "uri": "rtsp://192.168.1.25/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 90.0 }, + { "uri": "rtsp://192.168.1.26/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -130.0 }, + { "uri": "rtsp://192.168.1.27/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -90.0 }, + { "uri": "rtsp://192.168.1.28/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -135.0 }, + { "uri": "rtsp://192.168.1.29/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 125.0 }, + { "uri": "rtsp://192.168.1.30/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -50.0 }, + + { "uri": "rtsp://192.168.1.31/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 0.0 }, + { "uri": "rtsp://192.168.1.32/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 0.0 }, + { "uri": "rtsp://192.168.1.33/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 0.0 }, + { "uri": "rtsp://192.168.1.34/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": 135.0 }, + { "uri": "rtsp://192.168.1.35/user=admin&password=admin123&channel=1&stream=0.sdp?", "rotation": -45.0 }, + { "uri": "rtsp://192.168.1.36/user=admin&password=admin123&channel=1&stream=0.sdp?" } ] diff --git a/src/ffmpeg.rs b/src/ffmpeg.rs new file mode 100644 index 0000000..10ca78d --- /dev/null +++ b/src/ffmpeg.rs @@ -0,0 +1,32 @@ +use std::process::Command; + + +#[derive(Debug, Clone)] +pub struct FfmpegArgs { + pub mp4_path: String, + pub fps: u32, + pub width: u32, + pub height: u32, + pub interpolation: String, + pub output_directory: String, +} + +impl FfmpegArgs { + pub fn run(&self) -> std::io::Result<()> { + let output_pattern = format!("{}/%d.png", self.output_directory); + + let resolution = format!("{}x{}", self.width, self.height); + + let _ = Command::new("ffmpeg") + .arg("-i").arg(&self.mp4_path) + .arg("-r").arg(self.fps.to_string()) + .arg("-s").arg(&resolution) + .arg("-sws_flags").arg(&self.interpolation) + .arg("-vf").arg(format!("fps=fps={}", self.fps)) + .arg(&output_pattern) + .spawn()? + .wait()?; + + Ok(()) + } +} diff --git a/src/grid_view.rs b/src/grid_view.rs new file mode 100644 index 0000000..36f7dee --- /dev/null +++ b/src/grid_view.rs @@ -0,0 +1,145 @@ +use bevy::{ + prelude::*, + window::PrimaryWindow, +}; + +use crate::materials::foreground::ForegroundMaterial; + + +pub struct GridViewPlugin; +impl Plugin for GridViewPlugin { + fn build(&self, app: &mut App) { + app.init_resource::(); + app.add_systems(Update, draw_grid_view); + } +} + +#[derive(Debug, Clone)] +pub enum Element { + Image(Handle), + Alphablend(Handle), +} + +#[derive(Resource, Default)] +pub struct GridView { + pub source: Vec, +} + + +#[derive(Component, Default)] +pub struct GridViewParent; + + +fn draw_grid_view( + mut commands: Commands, + primary_window: Query< + &Window, + With + >, + grid_view: Res, + grid_view_parent: Query< + Entity, + With + >, +) { + if !grid_view.is_changed() { + return; + } + + for entity in grid_view_parent.iter() { + commands.entity(entity).despawn_recursive(); + } + + let window = primary_window.single(); + + let ( + columns, + rows, + _width, + _height, + ) = calculate_grid_dimensions( + window.width(), + window.height(), + grid_view.source.len(), + ); + + commands.spawn(NodeBundle { + style: Style { + display: Display::Grid, + width: Val::Percent(100.0), + height: Val::Percent(100.0), + grid_template_columns: RepeatedGridTrack::flex(columns as u16, 1.0), + grid_template_rows: RepeatedGridTrack::flex(rows as u16, 1.0), + ..default() + }, + background_color: BackgroundColor(Color::BLACK), + ..default() + }) + .insert(GridViewParent) + .with_children(|builder| { + grid_view.source.iter() + .for_each(|element| { + match element { + Element::Image(image) => { + builder.spawn(ImageBundle { + style: Style { + width: Val::Percent(100.0), + height: Val::Percent(100.0), + ..default() + }, + image: UiImage::new(image.clone()), + ..default() + }); + } + Element::Alphablend(material) => { + builder.spawn(MaterialNodeBundle { + style: Style { + width: Val::Percent(100.0), + height: Val::Percent(100.0), + ..default() + }, + material: material.clone(), + ..default() + }); + } + } + }); + }); +} + + +fn calculate_grid_dimensions( + window_width: f32, + window_height: f32, + num_streams: usize, +) -> (usize, usize, f32, f32) { + let window_aspect_ratio = window_width / window_height; + let stream_aspect_ratio: f32 = 16.0 / 9.0; + let mut best_layout = (1, num_streams); + let mut best_diff = f32::INFINITY; + let mut best_sprite_size = (0.0, 0.0); + + for columns in 1..=num_streams { + let rows = (num_streams as f32 / columns as f32).ceil() as usize; + let sprite_width = window_width / columns as f32; + let sprite_height = sprite_width / stream_aspect_ratio; + let total_height_needed = sprite_height * rows as f32; + let (final_sprite_width, final_sprite_height) = if total_height_needed > window_height { + let adjusted_sprite_height = window_height / rows as f32; + let adjusted_sprite_width = adjusted_sprite_height * stream_aspect_ratio; + (adjusted_sprite_width, adjusted_sprite_height) + } else { + (sprite_width, sprite_height) + }; + let grid_aspect_ratio = final_sprite_width * columns as f32 / (final_sprite_height * rows as f32); + let diff = (window_aspect_ratio - grid_aspect_ratio).abs(); + + if diff < best_diff { + best_diff = diff; + best_layout = (columns, rows); + best_sprite_size = (final_sprite_width, final_sprite_height); + } + } + + (best_layout.0, best_layout.1, best_sprite_size.0, best_sprite_size.1) +} diff --git a/src/lib.rs b/src/lib.rs index aeec5a2..cf9a052 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,15 @@ use bevy::prelude::*; +use bevy_ort::BevyOrtPlugin; -#[cfg(feature = "person_matting")] -pub mod matting; - +pub mod ffmpeg; +pub mod grid_view; pub mod materials; +pub mod matting; pub mod mp4; +pub mod person_detect; +pub mod pipeline; pub mod stream; +pub mod yolo; pub struct LightFieldPlugin { @@ -14,9 +18,15 @@ pub struct LightFieldPlugin { impl Plugin for LightFieldPlugin { fn build(&self, app: &mut App) { + app.add_plugins(BevyOrtPlugin); + + app.add_plugins(grid_view::GridViewPlugin); app.add_plugins(materials::StreamMaterialsPlugin); + app.add_plugins(person_detect::PersonDetectPlugin); + app.add_plugins(pipeline::PipelinePlugin); app.add_plugins(stream::RtspStreamPlugin { stream_config: self.stream_config.clone(), }); + app.add_plugins(yolo::YoloPlugin); } } diff --git a/src/matting.rs b/src/matting.rs index a7d769f..c4aad9c 100644 --- a/src/matting.rs +++ b/src/matting.rs @@ -4,11 +4,9 @@ use bevy::{ tasks::{block_on, futures_lite::future, AsyncComputeTaskPool, Task}, }; use bevy_ort::{ - BevyOrtPlugin, - inputs, models::modnet::{ - images_to_modnet_input, - modnet_output_to_luma_images, + Modnet, + modnet_inference, }, Onnx, }; @@ -45,9 +43,7 @@ impl MattingPlugin { impl Plugin for MattingPlugin { fn build(&self, app: &mut App) { - app.add_plugins(BevyOrtPlugin); app.register_type::(); - app.init_resource::(); app.insert_resource(self.max_inference_size.clone()); app.add_systems(Startup, load_modnet); app.add_systems(Update, matting_inference); @@ -55,17 +51,12 @@ impl Plugin for MattingPlugin { } -#[derive(Resource, Default)] -pub struct Modnet { - pub onnx: Handle, -} - fn load_modnet( asset_server: Res, mut modnet: ResMut, ) { - let modnet_handle: Handle = asset_server.load("modnet_photographic_portrait_matting.onnx"); + let modnet_handle: Handle = asset_server.load("models/modnet_photographic_portrait_matting.onnx"); modnet.onnx = modnet_handle; } @@ -107,7 +98,8 @@ fn matting_inference( .map(|(_, matted_stream)| { let input = images.get(matted_stream.input.clone()).unwrap(); let output = (matted_stream.output.clone(), matted_stream.material.clone()); - (input, output) + + (input.clone(), output) }) .unzip(); @@ -116,11 +108,6 @@ fn matting_inference( return; } - let input = images_to_modnet_input( - inputs.as_slice(), - inference_size.0.into(), - ); - if onnx_assets.get(&modnet.onnx).is_none() { return; } @@ -128,18 +115,20 @@ fn matting_inference( let onnx = onnx_assets.get(&modnet.onnx).unwrap(); let session_arc = onnx.session.clone(); + let inference_size = inference_size.0.into(); + let task = thread_pool.spawn(async move { + let inputs = inputs.iter().collect::>(); + let mask_images: Result, String> = (|| { let session_lock = session_arc.lock().map_err(|e| e.to_string())?; let session = session_lock.as_ref().ok_or("failed to get session from ONNX asset")?; - let input_values = inputs!["input" => input.view()].map_err(|e| e.to_string())?; - let outputs = session.run(input_values).map_err(|e| e.to_string()); - - let binding = outputs.ok().unwrap(); - let output_value: &ort::Value = binding.get("output").unwrap(); - - Ok(modnet_output_to_luma_images(output_value)) + Ok(modnet_inference( + session, + inputs.as_slice(), + inference_size, + )) })(); match mask_images { @@ -149,8 +138,8 @@ fn matting_inference( command_queue.push(move |world: &mut World| { world.resource_scope(|world, mut images: Mut>| { world.resource_scope(|_world, mut foreground_materials: Mut>| { - outputs.into_iter().zip(mask_images).for_each(|((output, material), mask_image)| { - images.insert(output, mask_image); + outputs.into_iter().zip(mask_images).for_each(|((mask, material), mask_image)| { + images.insert(mask, mask_image); foreground_materials.get_mut(&material).unwrap(); }); }); @@ -160,7 +149,7 @@ fn matting_inference( command_queue }, Err(error) => { - eprintln!("inference failed: {}", error); + error!("inference failed: {}", error); CommandQueue::default() } } diff --git a/src/person_detect.rs b/src/person_detect.rs new file mode 100644 index 0000000..45b78d4 --- /dev/null +++ b/src/person_detect.rs @@ -0,0 +1,184 @@ +use std::cmp::{max, min}; + +use bevy::prelude::*; +use image::{ImageBuffer, Luma}; + +use crate::{ + matting::MattedStream, + stream::StreamId, +}; + + +pub struct PersonDetectPlugin; + +impl Plugin for PersonDetectPlugin { + fn build(&self, app: &mut App) { + app.add_event::(); + app.add_systems(Update, detect_person); + } +} + + +#[derive(Component)] +pub struct DetectPersons; // TODO: add option for mask vs yolo detection + + +#[derive(Debug, Clone, Reflect, PartialEq)] +pub struct BoundingBox { + pub x: i32, + pub y: i32, + pub width: i32, + pub height: i32, +} + +#[derive(Event, Debug, Reflect, Clone)] +pub struct PersonDetectedEvent { + pub stream_id: StreamId, + pub bounding_box: BoundingBox, + pub mask_sum: f32, +} + + +fn detect_person( + mut ev_asset: EventReader>, + mut ev_person_detected: EventWriter, + person_detect_streams: Query<( + &MattedStream, + &DetectPersons, + )>, + images: Res>, +) { + for ev in ev_asset.read() { + if let AssetEvent::Modified { id } = ev { + for (matted_stream, _) in person_detect_streams.iter() { + if &matted_stream.output.id() == id { + let image = images.get(&matted_stream.output).unwrap(); + + let buffer = ImageBuffer::, Vec>::from_raw( + image.width(), + image.height(), + image.data.clone(), + ).unwrap(); + + let bounding_box = masked_bounding_box(&buffer); + let sum = sum_masked_pixels(&buffer); + + let masked_ratio = sum / (buffer.width() * buffer.height()) as f32; + let person_detected = masked_ratio > 0.14; + + if person_detected { + ev_person_detected.send(PersonDetectedEvent { + stream_id: matted_stream.stream_id, + bounding_box: bounding_box.unwrap(), + mask_sum: sum, + }); + } + } + } + } + } +} + + + +pub fn masked_bounding_box(buffer: &ImageBuffer, Vec>) -> Option { + let bounding_boxes = buffer.enumerate_pixels() + .filter_map(|(x, y, pixel)| { + if pixel.0[0] > 250 { + Some((x as i32, y as i32, x as i32, y as i32)) + } else { + None + } + }) + .reduce(|( + min_x1, + min_y1, + max_x1, + max_y1, + ), ( + min_x2, + min_y2, + max_x2, + max_y2, + )| { + ( + min(min_x1, min_x2), + min(min_y1, min_y2), + max(max_x1, max_x2), + max(max_y1, max_y2), + ) + }); + + bounding_boxes.map(|( + min_x, + min_y, + max_x, + max_y + )| { + BoundingBox { + x: min_x, + y: min_y, + width: max_x - min_x + 1, + height: max_y - min_y + 1, + } + }) +} + + +pub fn sum_masked_pixels(image: &ImageBuffer, Vec>) -> f32 { + image.pixels() + .map(|pixel| { + pixel.0[0] as f32 / 255.0 + }) + .sum() +} + + + +#[cfg(test)] +mod tests { + use super::*; + use image::{ImageBuffer, Luma}; + use approx::assert_relative_eq; + + + #[test] + fn test_masked_bounding_box() { + let width = 10; + let height = 10; + let mut img: ImageBuffer, Vec> = ImageBuffer::new(width, height); + + for x in 2..=5 { + for y in 2..=5 { + img.put_pixel(x, y, Luma([200])); + } + } + + let result = masked_bounding_box(&img).expect("expected a bounding box"); + + let expected = BoundingBox { + x:2, + y: 2, + width: 4, + height: 4, + }; + assert_eq!(result, expected, "the computed bounding box did not match the expected values."); + } + + + #[test] + fn test_sum_masked_pixels() { + let width = 4; + let height = 4; + let mut img: ImageBuffer, Vec> = ImageBuffer::new(width, height); + + img.put_pixel(0, 0, Luma([255])); + img.put_pixel(1, 0, Luma([127])); + img.put_pixel(2, 0, Luma([63])); + + let result = sum_masked_pixels(&img); + + let expected = (255.0 + 127.0 + 63.0) / 255.0; + assert_relative_eq!(result, expected); + } +} diff --git a/src/pipeline.rs b/src/pipeline.rs new file mode 100644 index 0000000..d576de7 --- /dev/null +++ b/src/pipeline.rs @@ -0,0 +1,981 @@ +use std::collections::HashMap; + +use bevy::{ + prelude::*, + render::{ + render_asset::RenderAssetUsages, + render_resource::Extent3d, + }, +}; +use bevy_ort::{ + models::{ + modnet::{ + modnet_inference, + Modnet, + ModnetPlugin + }, + yolo_v8::{ + yolo_inference, + BoundingBox, + Yolo, + YoloPlugin, + }, + }, + Onnx, +}; +use image::{ + DynamicImage, + GenericImageView, + ImageBuffer, + Luma, + Rgb, + Rgba, +}; +use imageproc::geometric_transformations::{ + rotate_about_center, + Interpolation, +}; +use png::Transformations; +use rayon::prelude::*; + +use crate::{ + ffmpeg::FfmpegArgs, + stream::{ + StreamId, + StreamDescriptors, + }, +}; + + +pub struct PipelinePlugin; +impl Plugin for PipelinePlugin { + fn build(&self, app: &mut App) { + app.add_plugins(( + ModnetPlugin, + YoloPlugin, + )); + app.add_systems( + Update, + ( + generate_raw_frames, + generate_rotated_frames, + generate_mask_frames, + generate_alphablend_frames, + generate_yolo_frames, + ) + ); + } +} + + +#[derive(Component, Reflect)] +pub struct PipelineConfig { + pub raw_frames: bool, + pub rotate_raw_frames: bool, + pub alphablend_frames: bool, + pub yolo: bool, // https://github.com/ultralytics/ultralytics + pub repair_frames: bool, // https://huggingface.co/docs/diffusers/en/optimization/onnx & https://github.com/bnm6900030/swintormer + pub upsample_frames: bool, // https://huggingface.co/ssube/stable-diffusion-x4-upscaler-onnx + pub mask_frames: bool, // https://github.com/ZHKKKe/MODNet + pub light_field_cameras: bool, // https://github.com/jasonyzhang/RayDiffusion + pub depth_maps: bool, // https://github.com/fabio-sim/Depth-Anything-ONNX + pub gaussian_cloud: bool, +} + +impl Default for PipelineConfig { + fn default() -> Self { + Self { + raw_frames: true, + rotate_raw_frames: true, + yolo: true, + alphablend_frames: true, + mask_frames: true, + upsample_frames: false, + repair_frames: false, + light_field_cameras: false, + depth_maps: false, + gaussian_cloud: false, + } + } +} + + +#[derive(Bundle, Default, Reflect)] +pub struct StreamSessionBundle { + pub config: PipelineConfig, + pub raw_streams: RawStreams, + pub session: Session, +} + +// TODO: use an entity saver to write Session and it's components (e.g. `0/session.ron`) + + +#[derive(Component, Default, Reflect)] +pub struct Session { + pub id: usize, + pub directory: String, +} + +impl Session { + pub fn new(directory: String) -> Self { + let id = get_next_session_id(&directory); + let directory = format!("{}/{}", directory, id); + std::fs::create_dir_all(&directory).unwrap(); + + Self { id, directory } + } + + pub fn from_id(id: usize, directory: String) -> Self { + let directory = format!("{}/{}", directory, id); + + Self { id, directory } + } +} + + +#[derive(Component, Default, Reflect)] +pub struct RawStreams { + pub streams: Vec, +} + +impl RawStreams { + pub fn load_from_session(session: &Session) -> Self { + let streams_directory = format!("{}/raw", session.directory); + + let streams = std::fs::read_dir(streams_directory) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file()) + .map(|entry| entry.path().to_str().unwrap().to_string()) + .collect::>(); + + Self { + streams, + } + } +} + + +// TODO: use the async task pool for all generate systems https://crates.io/crates/bevy-async-task +fn generate_raw_frames( + mut commands: Commands, + raw_streams: Query< + ( + Entity, + &PipelineConfig, + &RawStreams, + &Session, + ), + Without, + >, +) { + for ( + entity, + config, + raw_streams, + session, + ) in raw_streams.iter() { + if config.raw_frames { + let run_node = !RawFrames::exists(session); + let mut raw_frames = RawFrames::load_from_session(session); + + if run_node { + info!("generating raw frames for session {}", session.id); + + let frame_directory = format!("{}/frames", session.directory); + + raw_streams.streams.par_iter() + .for_each(|mp4_path| { + let stream_idx = std::path::Path::new(mp4_path).file_stem().unwrap().to_str().unwrap().parse::().unwrap(); + let output_directory = format!("{}/{}", frame_directory, stream_idx); + std::fs::create_dir_all(&output_directory).unwrap(); + + let _ = FfmpegArgs { + mp4_path: mp4_path.clone(), + fps: 5, + width: 1920, + height: 1080, + interpolation: "lanczos".to_string(), + output_directory, + }.run(); + }); + + raw_frames.reload(); + } else { + info!("raw frames already exist for session {}", session.id); + } + + commands.entity(entity).insert(raw_frames); + } + } +} + + +fn generate_rotated_frames( + mut commands: Commands, + descriptors: Res, + raw_frames: Query< + ( + Entity, + &PipelineConfig, + &RawFrames, + &Session, + ), + Without, + >, +) { + // TODO: create a caching/loading system wrapper over run_node interior + for ( + entity, + config, + raw_frames, + session, + ) in raw_frames.iter() { + if config.rotate_raw_frames { + let run_node = !RotatedFrames::exists(session); + let mut rotated_frames = RotatedFrames::load_from_session(session); + + if run_node { + let rotations: HashMap = descriptors.0.iter() + .enumerate() + .map(|(id, descriptor)| (StreamId(id), descriptor.rotation.unwrap_or_default())) + .collect(); + + info!("generating rotated frames for session {}", session.id); + + raw_frames.frames.iter() + .for_each(|(stream_id, frames)| { + let output_directory = format!("{}/{}", rotated_frames.directory, stream_id.0); + std::fs::create_dir_all(&output_directory).unwrap(); + + let frames = frames.par_iter() + .map(|frame| { + let frame_idx = std::path::Path::new(frame).file_stem().unwrap().to_str().unwrap(); + let output_path = format!("{}/{}.png", output_directory, frame_idx); + + rotate_image( + std::path::Path::new(frame), + std::path::Path::new(&output_path), + rotations[stream_id], + ).unwrap(); + + output_path + }) + .collect::>(); + + rotated_frames.frames.insert(*stream_id, frames); + }); + } else { + info!("rotated frames already exist for session {}", session.id); + } + + commands.entity(entity).insert(rotated_frames); + } + } +} + + +fn generate_mask_frames( + mut commands: Commands, + frames: Query< + ( + Entity, + &PipelineConfig, + &RotatedFrames, + &Session, + ), + Without, + >, + modnet: Res, + onnx_assets: Res>, +) { + for ( + entity, + config, + frames, + session, + ) in frames.iter() { + if config.mask_frames { + if onnx_assets.get(&modnet.onnx).is_none() { + return; + } + + let onnx = onnx_assets.get(&modnet.onnx).unwrap(); + let onnx_session_arc = onnx.session.clone(); + let onnx_session_lock = onnx_session_arc.lock().map_err(|e| e.to_string()).unwrap(); + let onnx_session = onnx_session_lock.as_ref().ok_or("failed to get session from ONNX asset").unwrap(); + + let run_node = !MaskFrames::exists(session); + let mut mask_frames = MaskFrames::load_from_session(session); + + if run_node { + info!("generating mask frames for session {}", session.id); + + frames.frames.keys() + .for_each(|stream_id| { + let output_directory = format!("{}/{}", mask_frames.directory, stream_id.0); + std::fs::create_dir_all(output_directory).unwrap(); + }); + + let mask_images = frames.frames.iter() + .map(|(stream_id, frames)| { + let frames = frames.iter() + .map(|frame| { + let mut decoder = png::Decoder::new(std::fs::File::open(frame).unwrap()); + decoder.set_transformations(Transformations::EXPAND | Transformations::ALPHA); + let mut reader = decoder.read_info().unwrap(); + let mut img_data = vec![0; reader.output_buffer_size()]; + let _ = reader.next_frame(&mut img_data).unwrap(); + + assert_eq!(reader.info().bytes_per_pixel(), 3); + + let width = reader.info().width; + let height = reader.info().height; + + // TODO: separate image loading and onnx inference (so the image loading result can be viewed in the pipeline grid view) + let image = Image::new( + Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + bevy::render::render_resource::TextureDimension::D2, + img_data, + bevy::render::render_resource::TextureFormat::Rgba8UnormSrgb, + RenderAssetUsages::all(), + ); + + let frame_idx = std::path::Path::new(frame).file_stem().unwrap().to_str().unwrap(); + + ( + frame_idx, + modnet_inference( + onnx_session, + &[&image], + Some((512, 512)), + ).pop().unwrap(), + ) + }) + .collect::>(); + + (stream_id, frames) + }) + .collect::>(); + + mask_images.iter() + .for_each(|(stream_id, frames)| { + let output_directory = format!("{}/{}", mask_frames.directory, stream_id.0); + let mask_paths = frames.iter() + .map(|(frame_idx, frame)| { + let path = format!("{}/{}.png", output_directory, frame_idx); + + let buffer = ImageBuffer::, Vec>::from_raw( + frame.width(), + frame.height(), + frame.data.clone(), + ).unwrap(); + + let _ = buffer.save(&path); + + path + }) + .collect::>(); + + mask_frames.frames.insert(**stream_id, mask_paths); + }); + } else { + info!("mask frames already exist for session {}", session.id); + } + + commands.entity(entity).insert(mask_frames); + } + } +} + + +fn generate_alphablend_frames( + mut commands: Commands, + session: Query< + ( + Entity, + &PipelineConfig, + &RotatedFrames, + &MaskFrames, + &Session, + ), + Without, + >, +) { + for ( + entity, + config, + rotated_frames, + mask_frames, + session, + ) in session.iter() { + if config.alphablend_frames { + let run_node = !AlphablendFrames::exists(session); + let mut alphablend_frames = AlphablendFrames::load_from_session(session); + + if run_node { + info!("generating alphablend frames for session {}", session.id); + + rotated_frames.frames.iter() + .for_each(|(stream_id, frames)| { + let output_directory = format!("{}/{}", alphablend_frames.directory, stream_id.0); + std::fs::create_dir_all(&output_directory).unwrap(); + + let frames = frames.par_iter() + .zip(mask_frames.frames.get(stream_id).unwrap()) + .map(|(frame, mask)| { + let frame_idx = std::path::Path::new(frame).file_stem().unwrap().to_str().unwrap(); + let output_path = format!("{}/{}.png", output_directory, frame_idx); + + alphablend_image( + std::path::Path::new(frame), + std::path::Path::new(mask), + std::path::Path::new(&output_path), + ).unwrap(); + + output_path + }) + .collect::>(); + + alphablend_frames.frames.insert(*stream_id, frames); + }); + } else { + info!("alphablend frames already exist for session {}", session.id); + } + + commands.entity(entity).insert(alphablend_frames); + } + } +} + + +fn generate_yolo_frames( + mut commands: Commands, + raw_frames: Query< + ( + Entity, + &PipelineConfig, + &RawFrames, + &Session, + ), + Without, + >, + yolo: Res, + onnx_assets: Res>, +) { + for ( + entity, + config, + raw_frames, + session, + ) in raw_frames.iter() { + if config.yolo { + if onnx_assets.get(&yolo.onnx).is_none() { + return; + } + + let onnx = onnx_assets.get(&yolo.onnx).unwrap(); + let onnx_session_arc = onnx.session.clone(); + let onnx_session_lock = onnx_session_arc.lock().map_err(|e| e.to_string()).unwrap(); + let onnx_session = onnx_session_lock.as_ref().ok_or("failed to get session from ONNX asset").unwrap(); + + let run_node = !YoloFrames::exists(session); + let mut yolo_frames = YoloFrames::load_from_session(session); + + if run_node { + info!("generating yolo frames for session {}", session.id); + + raw_frames.frames.keys() + .for_each(|stream_id| { + let output_directory = format!("{}/{}", yolo_frames.directory, stream_id.0); + std::fs::create_dir_all(output_directory).unwrap(); + }); + + // TODO: support async ort inference (re. progress bars) + let bounding_box_streams = raw_frames.frames.iter() + .map(|(stream_id, frames)| { + let frames = frames.iter() + .map(|frame| { + let mut decoder = png::Decoder::new(std::fs::File::open(frame).unwrap()); + decoder.set_transformations(Transformations::EXPAND | Transformations::ALPHA); + let mut reader = decoder.read_info().unwrap(); + let mut img_data = vec![0; reader.output_buffer_size()]; + let _ = reader.next_frame(&mut img_data).unwrap(); + + assert_eq!(reader.info().bytes_per_pixel(), 3); + + let width = reader.info().width; + let height = reader.info().height; + + // TODO: separate image loading and onnx inference (so the image loading result can be viewed in the pipeline grid view) + let image = Image::new( + Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + bevy::render::render_resource::TextureDimension::D2, + img_data, + bevy::render::render_resource::TextureFormat::Rgba8UnormSrgb, + RenderAssetUsages::all(), + ); + + let frame_idx = std::path::Path::new(frame).file_stem().unwrap().to_str().unwrap(); + + ( + frame_idx, + yolo_inference( + onnx_session, + &image, + 0.5, + ), + ) + }) + .collect::>(); + + (stream_id, frames) + }) + .collect::>(); + + bounding_box_streams.iter() + .for_each(|(stream_id, frames)| { + let output_directory = format!("{}/{}", yolo_frames.directory, stream_id.0); + let bounding_boxes = frames.iter() + .map(|(frame_idx, bounding_boxes)| { + let path = format!("{}/{}.json", output_directory, frame_idx); + + let _ = serde_json::to_writer(std::fs::File::create(path).unwrap(), bounding_boxes); + + bounding_boxes.clone() + }) + .collect::>(); + + yolo_frames.frames.insert(**stream_id, bounding_boxes); + }); + } else { + info!("yolo frames already exist for session {}", session.id); + } + + commands.entity(entity).insert(yolo_frames); + } + } +} + + +#[derive(Component, Default)] +pub struct AlphablendFrames { + pub frames: HashMap>, + pub directory: String, +} +impl AlphablendFrames { + pub fn load_from_session( + session: &Session, + ) -> Self { + let directory = format!("{}/alphablend", session.directory); + std::fs::create_dir_all(&directory).unwrap(); + + let mut alphablend_frames = Self { + frames: HashMap::new(), + directory, + }; + alphablend_frames.reload(); + + alphablend_frames + } + + pub fn reload(&mut self) { + std::fs::read_dir(&self.directory) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .map(|stream_dir| { + let stream_id = StreamId(stream_dir.path().file_name().unwrap().to_str().unwrap().parse::().unwrap()); + + let frames = std::fs::read_dir(stream_dir.path()).unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("png")) + .map(|entry| entry.path().to_str().unwrap().to_string()) + .collect::>(); + + (stream_id, frames) + }) + .for_each(|(stream_id, frames)| { + self.frames.insert(stream_id, frames); + }); + } + + pub fn exists( + session: &Session, + ) -> bool { + let output_directory = format!("{}/alphablend", session.directory); + std::fs::metadata(output_directory).is_ok() + } + + pub fn image(&self, _camera: usize, _frame: usize) -> Option { + todo!() + } +} + + + +// TODO: support loading maskframes -> images into a pipeline mask viewer + + +#[derive(Component, Default)] +pub struct RawFrames { + pub frames: HashMap>, + pub directory: String, +} +impl RawFrames { + pub fn load_from_session( + session: &Session, + ) -> Self { + let directory = format!("{}/frames", session.directory); + std::fs::create_dir_all(&directory).unwrap(); + + let mut raw_frames = Self { + frames: HashMap::new(), + directory, + }; + raw_frames.reload(); + + raw_frames + } + + pub fn reload(&mut self) { + std::fs::read_dir(&self.directory) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .map(|stream_dir| { + let stream_id = StreamId(stream_dir.path().file_name().unwrap().to_str().unwrap().parse::().unwrap()); + + let frames = std::fs::read_dir(stream_dir.path()).unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("png")) + .map(|entry| entry.path().to_str().unwrap().to_string()) + .collect::>(); + + (stream_id, frames) + }) + .for_each(|(stream_id, frames)| { + self.frames.insert(stream_id, frames); + }); + } + + pub fn exists( + session: &Session, + ) -> bool { + let output_directory = format!("{}/frames", session.directory); + std::fs::metadata(output_directory).is_ok() + } + + pub fn image(&self, _camera: usize, _frame: usize) -> Option { + todo!() + } +} + + +#[derive(Component, Default)] +pub struct YoloFrames { + pub frames: HashMap>>, + pub directory: String, +} +impl YoloFrames { + pub fn load_from_session( + session: &Session, + ) -> Self { + let directory = format!("{}/yolo_frames", session.directory); + std::fs::create_dir_all(&directory).unwrap(); + + let mut yolo_frames = Self { + frames: HashMap::new(), + directory, + }; + yolo_frames.reload(); + + yolo_frames + } + + pub fn reload(&mut self) { + std::fs::read_dir(&self.directory) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .map(|stream_dir| { + let stream_id = StreamId(stream_dir.path().file_name().unwrap().to_str().unwrap().parse::().unwrap()); + + let frames = std::fs::read_dir(stream_dir.path()).unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("json")) + .map(|entry| std::fs::File::open(entry.path()).unwrap()) + .map(|yolo_json_file| { + let bounding_boxes: Vec = serde_json::from_reader(&yolo_json_file).unwrap(); + + bounding_boxes + }) + .collect::>(); + + // TODO: parse the json at each frame path to get the bounding boxes + + (stream_id, frames) + }) + .for_each(|(stream_id, frames)| { + self.frames.insert(stream_id, frames); + }); + } + + pub fn write(&self) { + self.frames.iter() + .for_each(|(stream_id, frames)| { + let output_directory = format!("{}/{}", self.directory, stream_id.0); + std::fs::create_dir_all(&output_directory).unwrap(); + + frames.iter() + .enumerate() + .for_each(|(frame_idx, bounding_boxes)| { + let path = format!("{}/{}.json", output_directory, frame_idx); + let _ = serde_json::to_writer(std::fs::File::create(path).unwrap(), bounding_boxes); + }); + }); + } + + pub fn exists( + session: &Session, + ) -> bool { + let output_directory = format!("{}/yolo_frames", session.directory); + std::fs::metadata(output_directory).is_ok() + } + + pub fn image(&self, _camera: usize, _frame: usize) -> Option { + todo!() + } +} + + + +#[derive(Component, Default)] +pub struct RotatedFrames { + pub frames: HashMap>, + pub directory: String, +} +impl RotatedFrames { + pub fn load_from_session( + session: &Session, + ) -> Self { + let directory = format!("{}/rotated_frames", session.directory); + std::fs::create_dir_all(&directory).unwrap(); + + let mut raw_frames = Self { + frames: HashMap::new(), + directory, + }; + raw_frames.reload(); + + raw_frames + } + + pub fn reload(&mut self) { + std::fs::read_dir(&self.directory) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .map(|stream_dir| { + let stream_id = StreamId(stream_dir.path().file_name().unwrap().to_str().unwrap().parse::().unwrap()); + + let frames = std::fs::read_dir(stream_dir.path()).unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("png")) + .map(|entry| entry.path().to_str().unwrap().to_string()) + .collect::>(); + + (stream_id, frames) + }) + .for_each(|(stream_id, frames)| { + self.frames.insert(stream_id, frames); + }); + } + + pub fn exists( + session: &Session, + ) -> bool { + let output_directory = format!("{}/rotated_frames", session.directory); + std::fs::metadata(output_directory).is_ok() + } + + pub fn image(&self, _camera: usize, _frame: usize) -> Option { + todo!() + } +} + + + +#[derive(Component, Default, Reflect)] +pub struct MaskFrames { + pub frames: HashMap>, + pub directory: String +} +impl MaskFrames { + pub fn load_from_session( + session: &Session, + ) -> Self { + let directory = format!("{}/masks", session.directory); + std::fs::create_dir_all(&directory).unwrap(); + + let mut mask_frames = Self { + frames: HashMap::new(), + directory, + }; + mask_frames.reload(); + + mask_frames + } + + pub fn reload(&mut self) { + std::fs::read_dir(&self.directory) + .unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .map(|stream_dir| { + let stream_id = StreamId(stream_dir.path().file_name().unwrap().to_str().unwrap().parse::().unwrap()); + + let frames = std::fs::read_dir(stream_dir.path()).unwrap() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_file() && entry.path().extension().and_then(|s| s.to_str()) == Some("png")) + .map(|entry| entry.path().to_str().unwrap().to_string()) + .collect::>(); + + (stream_id, frames) + }) + .for_each(|(stream_id, frames)| { + self.frames.insert(stream_id, frames); + }); + } + + pub fn exists( + session: &Session, + ) -> bool { + let output_directory = format!("{}/masks", session.directory); + std::fs::metadata(output_directory).is_ok() + } + + pub fn image(&self, _camera: usize, _frame: usize) -> Option { + todo!() + } +} + + +#[derive(Default, Clone, Reflect)] +pub struct LightFieldCamera { + // TODO: intrinsics/extrinsics +} + +#[derive(Component, Default, Reflect)] +pub struct LightFieldCameras { + pub cameras: Vec, +} + + + +fn get_next_session_id(output_directory: &str) -> usize { + match std::fs::read_dir(output_directory) { + Ok(entries) => entries.filter_map(|entry| { + let entry = entry.ok()?; + if entry.path().is_dir() { + entry.file_name().to_string_lossy().parse::().ok() + } else { + None + } + }) + .max() + .map_or(0, |max_id| max_id + 1), + Err(_) => 0, + } +} + + +pub fn load_png( + image_path: &std::path::Path, +) -> Image { + let image = image::open(image_path).unwrap(); + let image = image.into_rgba8(); + let width = image.width(); + let height = image.height(); + + let image_bytes = image.into_raw(); + + Image::new( + Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + bevy::render::render_resource::TextureDimension::D2, + image_bytes, + bevy::render::render_resource::TextureFormat::Rgba8UnormSrgb, + RenderAssetUsages::all(), + ) +} + + +fn rotate_image( + image_path: &std::path::Path, + output_path: &std::path::Path, + angle: f32, +) -> image::ImageResult<()> { + if angle == 0.0 { + std::fs::copy(image_path, output_path)?; + return Ok(()); + } + + let dyn_img = image::open(image_path).unwrap(); + let (w, h) = dyn_img.dimensions(); + + let image_bytes = DynamicImage::into_bytes(dyn_img); + let image_buffer = ImageBuffer::, Vec>::from_vec( + w, + h, + image_bytes[..].to_vec(), + ).unwrap(); + + let radians = angle.to_radians(); + + let rotated_image: ImageBuffer::, Vec> = rotate_about_center( + &image_buffer, + radians, + Interpolation::Bilinear, + Rgb([0, 0, 0]), + ); + rotated_image.save(output_path)?; + + Ok(()) +} + + +fn alphablend_image( + image_path: &std::path::Path, + mask_path: &std::path::Path, + output_path: &std::path::Path, +) -> image::ImageResult<()> { + let img = image::open(image_path).unwrap(); + + let mask = image::open(mask_path).unwrap(); + let mask = mask.resize_exact(img.width(), img.height(), image::imageops::FilterType::Triangle); + + let mut output_img: ImageBuffer, Vec> = ImageBuffer::new(img.dimensions().0, img.dimensions().1); + + for (x, y, pixel) in img.pixels() { + let mask_pixel = mask.get_pixel(x, y).0[0]; + let mut img_pixel = pixel.0; + img_pixel[3] = mask_pixel; + output_img.put_pixel(x, y, Rgba(img_pixel)); + } + + output_img.save(output_path)?; + + Ok(()) +} diff --git a/src/stream.rs b/src/stream.rs index bc1588d..9eaaffd 100644 --- a/src/stream.rs +++ b/src/stream.rs @@ -4,7 +4,16 @@ use std::sync::{Arc, Mutex}; use anyhow::{bail, Error}; use bevy::{ prelude::*, - render::render_resource::Extent3d, + render::{ + render_asset::RenderAssetUsages, + render_resource::{ + Extent3d, + TextureDescriptor, + TextureDimension, + TextureFormat, + TextureUsages, + }, + }, }; use futures::TryStreamExt; use openh264::{ @@ -20,6 +29,7 @@ use retina::{ SessionOptions, SetupOptions, TcpTransportOptions, + UdpTransportOptions, Transport, }, codec::VideoFrame, @@ -27,15 +37,15 @@ use retina::{ use serde::{Deserialize, Serialize}; use tokio::{ fs::File, - runtime::{ - Handle, - Runtime, - }, + runtime::Handle, sync::mpsc, }; use url::Url; -use crate::mp4::Mp4Writer; +use crate::{ + mp4::Mp4Writer, + pipeline::Session as PipelineSession, +}; pub struct RtspStreamPlugin { @@ -45,16 +55,35 @@ pub struct RtspStreamPlugin { impl Plugin for RtspStreamPlugin { fn build(&self, app: &mut App) { let config = std::fs::File::open(&self.stream_config).unwrap(); - let stream_uris = serde_json::from_reader::<_, StreamUris>(config).unwrap(); + let stream_uris = serde_json::from_reader::<_, StreamDescriptors>(config).unwrap(); app .insert_resource(stream_uris) .init_resource::() + .add_systems(PreStartup, create_streams) .add_systems(Update, create_streams_from_descriptors) .add_systems(Update, apply_decode); } } +fn create_streams( + mut commands: Commands, + mut images: ResMut>, + stream_uris: Res, +) { + stream_uris.0.iter() + .enumerate() + .for_each(|(index, descriptor)| { + let rtsp_stream = RtspStreamHandle::new( + descriptor.clone(), + StreamId(index), + &mut images, + ); + + commands.spawn(rtsp_stream); + }); +} + fn create_streams_from_descriptors( mut commands: Commands, @@ -62,7 +91,7 @@ fn create_streams_from_descriptors( descriptors: Query< ( Entity, - &RtspStreamDescriptor, + &RtspStreamHandle, ), Without, >, @@ -78,7 +107,7 @@ fn create_streams_from_descriptors( pub fn apply_decode( mut images: ResMut>, - descriptors: Query<&RtspStreamDescriptor>, + descriptors: Query<&RtspStreamHandle>, ) { for descriptor in descriptors.iter() { let frame = descriptor.take_frame(); @@ -108,7 +137,7 @@ pub fn apply_decode( } -#[derive(Debug, Clone, Copy, PartialEq, Reflect)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Reflect)] pub struct StreamId(pub usize); #[derive(Debug)] @@ -118,23 +147,75 @@ pub enum RecordingCommand { } -#[derive(Component, Clone)] -pub struct RtspStreamDescriptor { +#[derive(Clone, Default, Debug, Serialize, Deserialize)] +pub enum StreamTransport { + #[default] + Tcp, + Udp, +} + +#[derive(Component, Clone, Default, Debug, Serialize, Deserialize)] +pub struct StreamDescriptor { pub uri: String, + + #[serde(default)] + pub transport: StreamTransport, + + pub visible: Option, + pub person_detection: Option, + + pub rotation: Option, +} + +#[derive(Resource, Clone, Debug, Default, Serialize, Deserialize)] +pub struct StreamDescriptors(pub Vec); + + +#[derive(Component, Clone)] +pub struct RtspStreamHandle { + pub descriptor: StreamDescriptor, pub id: StreamId, pub image: bevy::asset::Handle, latest_frame: Arc>>, recording_sender: Arc>>>, } -impl RtspStreamDescriptor { +impl RtspStreamHandle { pub fn new( - uri: String, + descriptor: StreamDescriptor, id: StreamId, - image: bevy::asset::Handle, + images: &mut Assets, ) -> Self { + let size = Extent3d { + width: 32, + height: 32, + ..default() + }; + + // TODO: use a default 'stream loading' texture + + let mut image = Image { + asset_usage: RenderAssetUsages::all(), + texture_descriptor: TextureDescriptor { + label: None, + size, + dimension: TextureDimension::D2, + format: TextureFormat::Rgba8UnormSrgb, + mip_level_count: 1, + sample_count: 1, + usage: TextureUsages::COPY_DST + | TextureUsages::TEXTURE_BINDING + | TextureUsages::RENDER_ATTACHMENT, + view_formats: &[TextureFormat::Rgba8UnormSrgb], + }, + ..default() + }; + image.resize(size); + + let image = images.add(image); + Self { - uri, + descriptor, id, image, latest_frame: Arc::new(Mutex::new(None)), @@ -166,13 +247,17 @@ struct Bgra8Frame { #[derive(Resource)] pub struct RtspStreamManager { - stream_descriptors: Arc>>, + stream_handles: Arc>>, handle: Handle, } impl FromWorld for RtspStreamManager { fn from_world(_world: &mut World) -> Self { - let runtime = Runtime::new().unwrap(); + + // TODO: upgrade to [bevy-tokio-tasks](https://github.com/EkardNT/bevy-tokio-tasks) to share tokio runtime between rtsp and inference - waiting on: https://github.com/pykeio/ort/pull/174 + let mut runtime = tokio::runtime::Builder::new_multi_thread(); + runtime.enable_all(); + let runtime = runtime.build().unwrap(); let handle = runtime.handle().clone(); std::thread::spawn(move || { @@ -184,7 +269,7 @@ impl FromWorld for RtspStreamManager { }); Self { - stream_descriptors: Arc::new(Mutex::new(vec![])), + stream_handles: Arc::new(Mutex::new(vec![])), handle, } } @@ -192,11 +277,11 @@ impl FromWorld for RtspStreamManager { impl RtspStreamManager { pub fn contains(&self, id: StreamId) -> bool { - self.stream_descriptors.lock().unwrap().iter().any(|s: &RtspStreamDescriptor| s.id == id) + self.stream_handles.lock().unwrap().iter().any(|s: &RtspStreamHandle| s.id == id) } pub fn add_stream(&self, stream: RtspStream) { - self.stream_descriptors.lock().unwrap().push(stream.descriptor.clone()); + self.stream_handles.lock().unwrap().push(stream.handle.clone()); self.handle.spawn(async move { let mut stream = stream; @@ -207,50 +292,72 @@ impl RtspStreamManager { }); } - pub fn start_recording(&self, output_directory: &str, prefix: &str) { - let stream_descriptors = self.stream_descriptors.lock().unwrap(); - for descriptor in stream_descriptors.iter() { - let filepath = format!("{}/{}_{}.mp4", output_directory, prefix, descriptor.id.0); + pub fn start_recording(&self, session: &PipelineSession) { + let output_directory = format!("{}/raw", session.directory); + std::fs::create_dir_all(&output_directory).unwrap(); + + let stream_handles = self.stream_handles.lock().unwrap(); + for descriptor in stream_handles.iter() { + let filename = format!("{}.mp4", descriptor.id.0); + let filepath = format!("{}/{}", output_directory, filename); let send_channel = descriptor.recording_sender.lock().unwrap(); + + if send_channel.is_none() { + warn!("no recording sender for stream {}", descriptor.id.0); + continue; + } + let sender_clone = send_channel.as_ref().unwrap().clone(); self.handle.block_on(async move { let file = File::create(&filepath).await.unwrap(); - sender_clone.send(RecordingCommand::StartRecording(file)).await.unwrap(); + let _ = sender_clone.send(RecordingCommand::StartRecording(file)).await; }); } } - pub fn stop_recording(&self) { - let stream_descriptors = self.stream_descriptors.lock().unwrap(); - for descriptor in stream_descriptors.iter() { + pub fn stop_recording(&self) -> Vec { + let mut filepaths = vec![]; + + let stream_handles = self.stream_handles.lock().unwrap(); + for descriptor in stream_handles.iter() { let send_channel = descriptor.recording_sender.lock().unwrap(); + + if send_channel.is_none() { + warn!("no recording sender for stream {}", descriptor.id.0); + continue; + } + let sender_clone = send_channel.as_ref().unwrap().clone(); self.handle.block_on(async move { - sender_clone.send(RecordingCommand::StopRecording).await.unwrap(); + let _ = sender_clone.send(RecordingCommand::StopRecording).await; }); + + filepaths.push(format!("{}.mp4", descriptor.id.0)); } + + filepaths } } pub struct RtspStream { - pub descriptor: RtspStreamDescriptor, + pub handle: RtspStreamHandle, decoder: Option, demuxed: Option, writer: Option>, } impl RtspStream { - pub fn new(descriptor: RtspStreamDescriptor) -> Self { + pub fn new(handle: RtspStreamHandle) -> Self { let api = openh264::OpenH264API::from_source(); let decoder = Decoder::new(api).ok(); Self { - descriptor, + handle, decoder, demuxed: None, writer: None, @@ -258,13 +365,13 @@ impl RtspStream { } async fn run(&mut self) -> Result<(), Box>{ - let (session, stream_idx) = create_session(&self.descriptor.uri).await?; + let (session, stream_idx) = create_session(&self.handle.descriptor).await?; self.demuxed = session.demuxed()?.into(); let (sender, mut receiver) = mpsc::channel(1); { - let mut send_channel = self.descriptor.recording_sender.lock().unwrap(); + let mut send_channel = self.handle.recording_sender.lock().unwrap(); *send_channel = sender.into(); } @@ -284,11 +391,11 @@ impl RtspStream { file, ).await.ok(); - println!("writing stream {}", self.descriptor.id.0); + info!("writing stream {}", self.handle.id.0); }, RecordingCommand::StopRecording => { if let Some(writer) = self.writer.take() { - println!("stopped recording stream {}", self.descriptor.id.0); + info!("stopped recording stream {}", self.handle.id.0); writer.finish().await.ok(); } }, @@ -304,6 +411,8 @@ impl RtspStream { } } + // TODO: enable/disable decoding based on whether the live frames are being used + let mut data = frame.into_data(); convert_h264(&mut data)?; @@ -315,7 +424,7 @@ impl RtspStream { let image_size = frame.dimension_rgb(); { - let mut locked_sink = self.descriptor.latest_frame.lock().unwrap(); + let mut locked_sink = self.handle.latest_frame.lock().unwrap(); match *locked_sink { Some(ref mut sink) => { assert_eq!(u32::from(sink.width), image_size.0 as u32, "frame width mismatch - stream size changes are not supported yet."); @@ -335,6 +444,8 @@ impl RtspStream { data, }; + // TODO: write streams into a frame texture array (stream, channel, width, height) + *locked_sink = Some(bgra); }, } @@ -359,11 +470,11 @@ impl RtspStream { } -async fn create_session(url: &str) -> Result< +async fn create_session(descriptor: &StreamDescriptor) -> Result< (Session, usize), Box > { - let parsed_url = Url::parse(url)?; + let parsed_url = Url::parse(&descriptor.uri)?; let username = parsed_url.username(); let password = parsed_url.password().unwrap_or(""); @@ -390,8 +501,10 @@ async fn create_session(url: &str) -> Result< options, ).await?; - let tcp_options = TcpTransportOptions::default(); - let transport = Transport::Tcp(tcp_options); + let transport = match descriptor.transport { + StreamTransport::Tcp => Transport::Tcp(TcpTransportOptions::default()), + StreamTransport::Udp => Transport::Udp(UdpTransportOptions::default()), + }; let video_stream_index = session.streams().iter().enumerate().find_map(|(i, s)| { if s.media() == "video" && s.encoding_name().to_uppercase() == "H264" { @@ -434,8 +547,3 @@ fn convert_h264(data: &mut [u8]) -> Result<(), Error> { Ok(()) } - - - -#[derive(Resource, Clone, Debug, Default, Reflect, Serialize, Deserialize)] -pub struct StreamUris(pub Vec); diff --git a/src/yolo.rs b/src/yolo.rs new file mode 100644 index 0000000..acb4c8b --- /dev/null +++ b/src/yolo.rs @@ -0,0 +1,22 @@ +use bevy::prelude::*; +use bevy_ort::{ + Onnx, + models::yolo_v8::Yolo, +}; + + +pub struct YoloPlugin; +impl Plugin for YoloPlugin { + fn build(&self, app: &mut App) { + app.init_resource::(); + app.add_systems(Startup, load_yolo); + } +} + +fn load_yolo( + asset_server: Res, + mut modnet: ResMut, +) { + let modnet_handle: Handle = asset_server.load("models/yolov8n.onnx"); + modnet.onnx = modnet_handle; +} diff --git a/tools/viewer.rs b/tools/viewer.rs index b77ce81..075010d 100644 --- a/tools/viewer.rs +++ b/tools/viewer.rs @@ -15,7 +15,7 @@ use bevy::{ TextureUsages, }, }, - window::PrimaryWindow, + time::Stopwatch, }; use bevy_args::{ parse_args, @@ -24,23 +24,57 @@ use bevy_args::{ Parser, Serialize, }; +use clap::ValueEnum; use bevy_light_field::{ - LightFieldPlugin, + grid_view::{ + Element, + GridView + }, materials::foreground::ForegroundMaterial, + matting::{ + MattedStream, + MattingPlugin, + }, + person_detect::{ + DetectPersons, + PersonDetectedEvent, + }, + pipeline::{ + load_png, + AlphablendFrames, + MaskFrames, + PipelineConfig, + RawFrames, + RawStreams, + RotatedFrames, + Session, + StreamSessionBundle, + }, stream::{ - RtspStreamDescriptor, + RtspStreamHandle, RtspStreamManager, - StreamId, - StreamUris, }, + LightFieldPlugin, }; -#[cfg(feature = "person_matting")] -use bevy_light_field::matting::{ - MattedStream, - MattingPlugin, -}; + +#[derive( + Debug, + Default, + Clone, + Serialize, + Deserialize, + ValueEnum, +)] +pub enum OfflineAnnotation { + Raw, + Rotated, + Mask, + #[default] + Alphablend, + Yolo, +} #[derive( @@ -59,6 +93,9 @@ pub struct LightFieldViewer { #[arg(long, default_value = "false")] pub show_fps: bool, + #[arg(long, default_value = "true")] + pub automatic_recording: bool, + #[arg(long, default_value = "false")] pub fullscreen: bool, @@ -67,13 +104,17 @@ pub struct LightFieldViewer { #[arg(long, default_value = "1080.0")] pub height: f32, - #[arg(long, default_value = "512")] + #[arg(long, default_value = "1024")] pub max_matting_width: u32, - #[arg(long, default_value = "512")] + #[arg(long, default_value = "1024")] pub max_matting_height: u32, - #[arg(long, default_value = "false")] - pub extract_foreground: bool, + #[arg(long)] + pub session_id: Option, + #[arg(long)] + pub annotation: Option, + #[arg(long)] + pub frame: Option, } @@ -81,6 +122,8 @@ pub struct LightFieldViewer { fn main() { let args = parse_args::(); + let online = args.session_id.is_none(); + let mode = if args.fullscreen { bevy::window::WindowMode::BorderlessFullscreen } else { @@ -108,27 +151,58 @@ fn main() { LightFieldPlugin { stream_config: args.config.clone(), }, - - #[cfg(feature = "person_matting")] MattingPlugin::new(( args.max_matting_width, args.max_matting_height, )), )) - .add_systems(Startup, create_streams) .add_systems(Startup, setup_camera) - .add_systems( - Update, - ( - press_esc_close, - press_r_start_recording, - press_s_stop_recording + .add_systems(Update, press_esc_close); + + if online { + app + .init_resource::() + .add_systems( + Startup, + ( + create_mask_streams, + ), ) - ); + .add_systems( + PostStartup, + ( + setup_live_gridview, + ), + ) + .add_systems( + Update, + ( + automatic_recording, + press_r_start_recording, + press_s_stop_recording + ), + ); + } else { + app + .insert_resource(FrameIndex(args.frame.unwrap_or_default())) + .add_systems( + Startup, + ( + select_session_from_args, + ), + ) + .add_systems( + Update, + ( + offline_viewer, + press_arrow_key_frame_navigation, + ), + ); + } if args.show_fps { app.add_plugins(FrameTimeDiagnosticsPlugin); - app.add_systems(Startup, fps_display_setup.after(create_streams)); + app.add_systems(PostStartup, fps_display_setup.after(setup_live_gridview)); app.add_systems(Update, fps_update_system); } @@ -136,158 +210,95 @@ fn main() { } -fn create_streams( +fn create_mask_streams( mut commands: Commands, mut images: ResMut>, - primary_window: Query<&Window, With>, - args: Res, mut foreground_materials: ResMut>, - stream_uris: Res, + args: Res, + input_streams: Query< + ( + Entity, + &RtspStreamHandle, + ), + Without + >, ) { - let window = primary_window.single(); - let elements = stream_uris.0.len(); - - let ( - columns, - rows, - _sprite_width, - _sprite_height, - ) = calculate_grid_dimensions( - window.width(), - window.height(), - elements, - ); - let size = Extent3d { width: 32, height: 32, ..default() }; - let input_images: Vec> = stream_uris.0.iter() - .enumerate() - .map(|(index, url)| { - let entity = commands.spawn_empty().id(); - - let mut image = Image { + input_streams.iter() + .for_each(|(entity, stream)| { + let mut mask_image = Image { asset_usage: RenderAssetUsages::all(), texture_descriptor: TextureDescriptor { label: None, size, dimension: TextureDimension::D2, - format: TextureFormat::Rgba8UnormSrgb, + format: TextureFormat::R8Unorm, mip_level_count: 1, sample_count: 1, usage: TextureUsages::COPY_DST | TextureUsages::TEXTURE_BINDING | TextureUsages::RENDER_ATTACHMENT, - view_formats: &[TextureFormat::Rgba8UnormSrgb], + view_formats: &[TextureFormat::R8Unorm], }, ..default() }; - image.resize(size); - - let image = images.add(image); - let image_clone = image.clone(); - - let rtsp_stream = RtspStreamDescriptor::new( - url.to_string(), - StreamId(index), - image, - ); + mask_image.resize(size); + let mask_image = images.add(mask_image); - commands.entity(entity).insert(rtsp_stream); - - image_clone - }) - .collect(); - - let mask_images = input_images.iter() - .enumerate() - .map(|(index, image)| { - let mut mask_images = Image { - asset_usage: RenderAssetUsages::all(), - texture_descriptor: TextureDescriptor { - label: None, - size, - dimension: TextureDimension::D2, - format: TextureFormat::Rgba8UnormSrgb, // TODO: use R8 format - mip_level_count: 1, - sample_count: 1, - usage: TextureUsages::COPY_DST - | TextureUsages::TEXTURE_BINDING - | TextureUsages::RENDER_ATTACHMENT, - view_formats: &[TextureFormat::Rgba8UnormSrgb], - }, - ..default() - }; - mask_images.resize(size); - let mask_image = images.add(mask_images); - - let mut material = None; - - #[cfg(feature = "person_matting")] - if args.extract_foreground { + if args.automatic_recording && stream.descriptor.person_detection.unwrap_or_default() { let foreground_mat = foreground_materials.add(ForegroundMaterial { - input: image.clone(), + input: stream.image.clone(), mask: mask_image.clone(), }); - commands.spawn(MattedStream { - stream_id: StreamId(index), - input: image.clone(), - output: mask_image.clone(), - material: foreground_mat.clone(), - }); - - material = foreground_mat.into(); + commands.entity(entity) + .insert(MattedStream { + stream_id: stream.id, + input: stream.image.clone(), + output: mask_image.clone(), + material: foreground_mat, + }) + .insert(DetectPersons); } + }); +} - (mask_image, material) - }) + +fn setup_live_gridview( + mut grid_view: ResMut, + input_streams: Query<( + Entity, + &RtspStreamHandle, + )>, + person_detection_stream: Query< + ( + Entity, + &MattedStream, + ), + With, + >, +) { + let visible_input_streams = input_streams.iter() + .filter(|(_, stream)| stream.descriptor.visible.unwrap_or_default()) .collect::>(); - commands.spawn(NodeBundle { - style: Style { - display: Display::Grid, - width: Val::Percent(100.0), - height: Val::Percent(100.0), - grid_template_columns: RepeatedGridTrack::flex(columns as u16, 1.0), - grid_template_rows: RepeatedGridTrack::flex(rows as u16, 1.0), - ..default() - }, - background_color: BackgroundColor(Color::BLACK), - ..default() - }) - .with_children(|builder| { - input_images.iter() - .zip(mask_images.iter()) - .for_each(|(input, (_mask, material))| { - if args.extract_foreground { - builder.spawn(MaterialNodeBundle { - style: Style { - width: Val::Percent(100.0), - height: Val::Percent(100.0), - ..default() - }, - material: material.clone().unwrap(), - ..default() - }); - } else { - builder.spawn(ImageBundle { - style: Style { - width: Val::Percent(100.0), - height: Val::Percent(100.0), - ..default() - }, - image: UiImage::new(input.clone()), - ..default() - }); - } - }); - }); + let grid_elements = visible_input_streams.iter() + .map(|(_, input_stream)| Element::Image(input_stream.image.clone())) + .chain( + person_detection_stream.iter() + .map(|(_, matted_stream) | Element::Alphablend(matted_stream.material.clone())) + ) + .collect::>(); + + grid_view.source = grid_elements; } + fn setup_camera( mut commands: Commands, ) { @@ -308,111 +319,249 @@ fn press_esc_close( } } + +fn select_session_from_args( + mut commands: Commands, + args: Res, +) { + if args.session_id.is_none() { + return; + } + + let session = Session::from_id( + args.session_id.unwrap(), + "capture".to_string(), + ); + let raw_streams = RawStreams::load_from_session(&session); + + commands.spawn( + StreamSessionBundle { + session, + raw_streams, + config: PipelineConfig::default(), + }, + ); +} + + +#[derive(Resource, Default)] +struct FrameIndex(usize); + +fn offline_viewer( + asset_server: Res, + mut grid_view: ResMut, + frame_index: Res, + args: Res, + session: Query< + ( + Entity, + &PipelineConfig, + &RawFrames, + &RotatedFrames, + &MaskFrames, + &AlphablendFrames, + &Session, + ), + >, + mut complete: Local, +) { + if session.is_empty() { + return; + } + + if !frame_index.is_changed() && *complete { + return; + } + + let session = session.iter().next().unwrap(); + + let mut frames = match args.annotation.clone().unwrap_or_default() { + OfflineAnnotation::Raw => &session.2.frames, + OfflineAnnotation::Rotated => &session.3.frames, + OfflineAnnotation::Mask => &session.4.frames, + OfflineAnnotation::Alphablend => &session.5.frames, + OfflineAnnotation::Yolo => unimplemented!(), + }.iter() + .map(|(stream_id, frames)| { + let mut sorted_frames = frames.clone(); + sorted_frames.sort_by(|a, b| { + let stem_a = std::path::Path::new(a).file_stem().unwrap().to_str().unwrap(); + let stem_b = std::path::Path::new(b).file_stem().unwrap().to_str().unwrap(); + + let a_idx = stem_a.parse::().unwrap(); + let b_idx = stem_b.parse::().unwrap(); + + a_idx.cmp(&b_idx) + }); + + (stream_id, sorted_frames[frame_index.0].clone()) + }) + .collect::>(); + + frames.sort_by(|a, b| a.0.0.partial_cmp(&b.0.0).unwrap()); + + let frames = frames.iter() + .map(|(_stream_id, frame)| { + let image = load_png(std::path::Path::new(frame)); + + Element::Image(asset_server.add(image)) + }) + .collect::>(); + + grid_view.source = frames; + + *complete = true; +} + + + +fn automatic_recording( + mut commands: Commands, + time: Res