From a8f4f4524663fdc26096b8dbfa0633fce40797d6 Mon Sep 17 00:00:00 2001 From: lucy66hm Date: Thu, 2 Jan 2025 19:20:12 +0000 Subject: [PATCH] Add geneate proto workflow --- tools/src/generate_proto_py/Pipfile | 13 + tools/src/generate_proto_py/Pipfile.lock | 349 ++++++++++++++++++ .../generate_proto_py/generate_proto_wf.py | 149 ++++++++ tools/src/generate_proto_py/preprocessing.py | 92 +++++ .../generate_proto_py/fixtures/animal.yaml | 52 +++ .../generate_proto_py/fixtures/expected.yaml | 43 +++ .../generate_proto_py/preprocessing_test.py | 45 +++ 7 files changed, 743 insertions(+) create mode 100644 tools/src/generate_proto_py/Pipfile create mode 100644 tools/src/generate_proto_py/Pipfile.lock create mode 100644 tools/src/generate_proto_py/generate_proto_wf.py create mode 100644 tools/src/generate_proto_py/preprocessing.py create mode 100644 tools/tests/generate_proto_py/fixtures/animal.yaml create mode 100644 tools/tests/generate_proto_py/fixtures/expected.yaml create mode 100644 tools/tests/generate_proto_py/preprocessing_test.py diff --git a/tools/src/generate_proto_py/Pipfile b/tools/src/generate_proto_py/Pipfile new file mode 100644 index 000000000..ca80b84eb --- /dev/null +++ b/tools/src/generate_proto_py/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +gitpython = "*" +pyyaml = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/tools/src/generate_proto_py/Pipfile.lock b/tools/src/generate_proto_py/Pipfile.lock new file mode 100644 index 000000000..d4ea913d8 --- /dev/null +++ b/tools/src/generate_proto_py/Pipfile.lock @@ -0,0 +1,349 @@ +{ + "_meta": { + "hash": { + "sha256": "170ece3aba513bf88b39e78b9f2db226f5dfc0dde819f7efb33e34d149bb1e21" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" + ], + "markers": "python_version >= '3.6'", + "version": "==2024.12.14" + }, + "charset-normalizer": { + "hashes": [ + "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621", + "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", + "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", + "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", + "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", + "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", + "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", + "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d", + "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", + "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", + "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", + "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", + "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab", + "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be", + "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", + "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", + "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0", + "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2", + "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62", + "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62", + "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", + "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", + "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", + "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", + "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455", + "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858", + "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", + "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", + "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", + "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", + "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", + "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea", + "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", + "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", + "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", + "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", + "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd", + "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", + "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242", + "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee", + "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", + "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", + "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51", + "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", + "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8", + "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", + "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613", + "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742", + "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", + "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", + "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", + "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", + "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", + "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", + "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", + "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", + "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417", + "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", + "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", + "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca", + "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa", + "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", + "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149", + "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41", + "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574", + "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0", + "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f", + "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", + "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654", + "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3", + "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19", + "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", + "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578", + "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", + "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", + "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51", + "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", + "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", + "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a", + "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", + "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade", + "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", + "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", + "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6", + "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", + "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", + "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6", + "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2", + "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12", + "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf", + "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", + "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7", + "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", + "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", + "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b", + "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", + "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", + "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4", + "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", + "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", + "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a", + "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748", + "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", + "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", + "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.4.0" + }, + "gitdb": { + "hashes": [ + "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4", + "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b" + ], + "markers": "python_version >= '3.7'", + "version": "==4.0.11" + }, + "gitpython": { + "hashes": [ + "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c", + "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==3.1.43" + }, + "idna": { + "hashes": [ + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.10" + }, + "lxml": { + "hashes": [ + "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e", + "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229", + "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3", + "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5", + "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70", + "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15", + "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002", + "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd", + "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22", + "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf", + "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22", + "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832", + "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727", + "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e", + "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30", + "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f", + "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f", + "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51", + "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4", + "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de", + "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875", + "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42", + "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e", + "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6", + "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391", + "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc", + "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b", + "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237", + "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4", + "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86", + "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f", + "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a", + "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8", + "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f", + "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903", + "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03", + "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e", + "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99", + "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7", + "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab", + "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d", + "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22", + "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492", + "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b", + "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3", + "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be", + "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469", + "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f", + "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a", + "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c", + "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a", + "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4", + "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94", + "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442", + "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b", + "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84", + "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c", + "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9", + "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1", + "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be", + "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367", + "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e", + "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21", + "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa", + "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16", + "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d", + "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe", + "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83", + "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba", + "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040", + "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763", + "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8", + "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff", + "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2", + "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a", + "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b", + "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce", + "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c", + "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577", + "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8", + "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71", + "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512", + "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540", + "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f", + "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2", + "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a", + "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce", + "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e", + "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2", + "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27", + "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1", + "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d", + "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1", + "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330", + "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920", + "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99", + "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff", + "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18", + "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff", + "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c", + "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179", + "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080", + "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19", + "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d", + "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70", + "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32", + "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a", + "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2", + "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79", + "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3", + "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5", + "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f", + "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d", + "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3", + "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b", + "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753", + "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9", + "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957", + "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033", + "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb", + "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656", + "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab", + "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b", + "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d", + "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd", + "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859", + "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11", + "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c", + "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a", + "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005", + "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654", + "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80", + "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e", + "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec", + "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7", + "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965", + "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945", + "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8" + ], + "markers": "python_version >= '3.6'", + "version": "==5.3.0" + }, + "python-maven": { + "hashes": [ + "sha256:3aee24704bebb6e65d92932040e77ba27fc9f9dd4e430444134e31952f7b88c7", + "sha256:c683e238e2d6728c1b3091dcce7cb67b7fba21789e92822923047a402582f6e7" + ], + "index": "pypi", + "version": "==0.0.3" + }, + "requests": { + "hashes": [ + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.3" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "smmap": { + "hashes": [ + "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62", + "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da" + ], + "markers": "python_version >= '3.7'", + "version": "==5.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", + "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" + ], + "markers": "python_version >= '3.8'", + "version": "==2.2.3" + } + }, + "develop": {} +} diff --git a/tools/src/generate_proto_py/generate_proto_wf.py b/tools/src/generate_proto_py/generate_proto_wf.py new file mode 100644 index 000000000..cf2432101 --- /dev/null +++ b/tools/src/generate_proto_py/generate_proto_wf.py @@ -0,0 +1,149 @@ +import os +import subprocess +import tempfile +from preprocessing import preprocess_openapi +from git import Repo +import argparse +import yaml +import re + +REPO_URL = "https://github.com/lucy66hw/openapi-generator.git" + +def clone_build(repo_url, branch_name="master"): + """ + Clones the repository, builds it with maven, and runs the generated JAR. + + Args: + repo_url (str): URL of the Git repository to clone. + branch_name (str): Branch to checkout. + """ + clone_dir = tempfile.mkdtemp(dir=temp_dir) + + try: + print(f"Cloning repository into: {clone_dir}") + repo = Repo.clone_from(repo_url, clone_dir) + print(f"Checking out branch: {branch_name}") + repo.git.checkout(branch_name) + + print("Building the project with Maven...") + subprocess.run(["./mvnw", "clean", "install"], cwd=clone_dir, check=True) + + jar_path = os.path.join(clone_dir, "modules", "openapi-generator-cli", "target", "openapi-generator-cli.jar") + if not os.path.exists(jar_path): + raise FileNotFoundError(f"JAR file not found: {jar_path}") + + return jar_path + + except FileNotFoundError as fnf_error: + print(f"File error: {fnf_error}") + raise + except subprocess.CalledProcessError as proc_error: + print(f"Process error during Maven or JAR execution: {proc_error}") + raise + except Exception as e: + print(f"Unexpected error: {e}") + raise + +def convert_proto(jar_path, jar_args=None): + jar_args = jar_args or [] + print(f"Running the JAR: {jar_path}") + subprocess.run(["java", "-jar", jar_path] + jar_args, check=True) + +def run_merger_script(ts_file, source, output): + try: + result = subprocess.run( + ["ts-node", ts_file, "--source", source, "--output", output], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True + ) + print("Output from TypeScript script:") + print(result.stdout) + except subprocess.CalledProcessError as e: + print("Error while running TypeScript script:") + print(e.stderr) + +def get_git_root(current_path="."): + try: + repo = Repo(current_path, search_parent_directories=True) + return repo.git.rev_parse("--show-toplevel") + except Exception as e: + print(f"Error determining Git root: {e}") + raise + +def get_openapi_paths(openapi_file): + try: + with open(openapi_file, 'r') as file: + spec = yaml.safe_load(file) + + paths = spec.get("paths", {}) + + return list(paths.keys()) + except Exception as e: + print(f"Error reading OpenAPI file {openapi_file}: {e}") + return [] + +def sanitize_path_name(path): + parts = path.split("/") + + sanitized_parts = [re.sub(r'\W+', '', part).strip("_") for part in parts] + + snake_case = "_".join(filter(None, sanitized_parts)) + if snake_case == "": + return "root" + return snake_case.lower().strip("_") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Process OpenSearch API specification.") + parser.add_argument( + "-s", "--selected-paths", + nargs="+", + default=None, + help="API paths to convert proto." + ) + + parser.add_argument( + "-b", "--branch-name", + type=str, + default="master", + help="The branch name to checkout." + ) + + args = parser.parse_args() + + current_path = os.getcwd() + git_root = get_git_root(current_path) + + merger_file = os.path.join(git_root, "tools", "src", "merger", "merge.ts") + spec_source = os.path.join(git_root, "spec") + proto_output_dir = os.path.join(git_root, "proto") + + with tempfile.TemporaryDirectory() as temp_dir: + # 1, merger + merger_output = os.path.join(temp_dir, "build", "opensearch-openapi.yaml") + run_merger_script(merger_file, spec_source, merger_output) + + + # 2, build tools + jar_path = clone_build(REPO_URL, args.branch_name) + + + selected_path = args.selected_paths or get_openapi_paths(merger_output) + + for sp in selected_path: + # 3. preprocess openapi yaml file + path_name = sanitize_path_name(sp) + selected_output = os.path.join(temp_dir, "build", path_name, "filtered-opensearch-openapi.yaml") + preprocess_openapi(merger_output, selected_output, sp) + proto_out = os.path.join(proto_output_dir, path_name) + + # 4. run tooling command + JAR_ARGS = [ + "generate", + "-i", selected_output, + "-g", "protobuf-schema", + "-o", proto_out, + "--additional-properties", "numberedFieldNumberList=true,startEnumsWithUnknown=true" + ] + + convert_proto(jar_path, JAR_ARGS) \ No newline at end of file diff --git a/tools/src/generate_proto_py/preprocessing.py b/tools/src/generate_proto_py/preprocessing.py new file mode 100644 index 000000000..bf1222fd8 --- /dev/null +++ b/tools/src/generate_proto_py/preprocessing.py @@ -0,0 +1,92 @@ +import yaml +import os + + +def collect_refs(item, used_refs, visited_refs, spec): + if isinstance(item, dict): + for key, value in item.items(): + if key == '$ref': + used_refs.add(value) + resolve_nested_ref(value, used_refs, visited_refs, spec) + else: + collect_refs(value, used_refs, visited_refs, spec) + elif isinstance(item, list): + for i in item: + collect_refs(i, used_refs, visited_refs, spec) + + +def resolve_nested_ref(ref, used_refs, visited_refs, spec): + if ref in visited_refs: + return + + visited_refs.add(ref) + + if ref.startswith("#/components/"): + ref_parts = ref.split('/') + schema = ref_parts[2] + component_name = ref_parts[3] + + if schema in spec['components'] and component_name in spec['components'][schema]: + component = spec['components'][schema][component_name] + collect_refs(component, used_refs, visited_refs, spec) + +def rename_component(name): + if '___' in name: + return name.split('___')[-1] + return name + + +def update_refs(item, rename_component): + if isinstance(item, dict): + for key, value in item.items(): + if key == '$ref' and value.startswith("#/components/"): + ref_parts = value.split('/') + schema = ref_parts[2] + component_name = ref_parts[3] + new_name = rename_component(component_name) + item[key] = f"#/components/{schema}/{new_name}" + else: + update_refs(value, rename_component) + elif isinstance(item, list): + for i in item: + update_refs(i, rename_component) + + +def preprocess_openapi(input_file, output_file, selected_path): + with open(input_file, 'r') as f: + spec = yaml.safe_load(f) + + if selected_path in spec['paths']: + filtered_paths = {selected_path: spec['paths'][selected_path]} + else: + raise KeyError(f"The selected path '{selected_path}' is not found in the OpenAPI spec.") + spec['paths'] = filtered_paths + + used_refs = set() + visited_refs = set() + + for path_item in filtered_paths.values(): + for method, operation in path_item.items(): + if isinstance(operation, dict): + collect_refs(operation, used_refs, visited_refs, spec) + + components = spec.get('components', {}) + filtered_components = {} + + for component_type, component_items in components.items(): + renamed_items = {} + for name, details in component_items.items(): + if f"#/components/{component_type}/{name}" in used_refs: + new_name = rename_component(name) + renamed_items[new_name] = details + filtered_components[component_type] = renamed_items + + spec['components'] = filtered_components + + update_refs(spec, rename_component) + + os.makedirs(os.path.dirname(output_file), exist_ok=True) + with open(output_file, 'w') as f: + yaml.dump(spec, f, default_flow_style=False) + + print(f"Filtered spec saved to {output_file}") diff --git a/tools/tests/generate_proto_py/fixtures/animal.yaml b/tools/tests/generate_proto_py/fixtures/animal.yaml new file mode 100644 index 000000000..59586bd1e --- /dev/null +++ b/tools/tests/generate_proto_py/fixtures/animal.yaml @@ -0,0 +1,52 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + /pets: + get: + parameters: + - $ref: '#/components/parameters/parameter___Cat' + /animals: + get: + parameters: + - $ref: '#/components/parameters/parameter___Dog' + responses: + '200': + description: A list of animals + content: + application/json: + schema: + $ref: "#/components/schemas/animals___Animal" +components: + parameters: + parameter___Cat: + name: a + schema: + type: string + in: query + parameter___Dog: + name: b + schema: + type: string + in: query + schemas: + actions___Bark: + type: string + actions___Meow: + type: string + animals___Animal: + oneOf: + - $ref: '#/components/schemas/animals___Dog' + - $ref: '#/components/schemas/animals___Cat' + animals___Cat: + type: object + properties: + meow: + $ref: '#/components/schemas/actions___Meow' + animals___Dog: + type: object + properties: + bark: + $ref: '#/components/schemas/actions___Bark' \ No newline at end of file diff --git a/tools/tests/generate_proto_py/fixtures/expected.yaml b/tools/tests/generate_proto_py/fixtures/expected.yaml new file mode 100644 index 000000000..a2c8a32b9 --- /dev/null +++ b/tools/tests/generate_proto_py/fixtures/expected.yaml @@ -0,0 +1,43 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + /animals: + get: + parameters: + - $ref: '#/components/parameters/Dog' + responses: + '200': + description: A list of animals + content: + application/json: + schema: + $ref: "#/components/schemas/Animal" +components: + parameters: + Dog: + name: b + schema: + type: string + in: query + schemas: + Bark: + type: string + Meow: + type: string + Animal: + oneOf: + - $ref: '#/components/schemas/Dog' + - $ref: '#/components/schemas/Cat' + Cat: + type: object + properties: + meow: + $ref: '#/components/schemas/Meow' + Dog: + type: object + properties: + bark: + $ref: '#/components/schemas/Bark' diff --git a/tools/tests/generate_proto_py/preprocessing_test.py b/tools/tests/generate_proto_py/preprocessing_test.py new file mode 100644 index 000000000..815ee2b57 --- /dev/null +++ b/tools/tests/generate_proto_py/preprocessing_test.py @@ -0,0 +1,45 @@ +import unittest +import yaml +import os +from tools.src.generate_proto_py.preprocessing import preprocess_openapi +import tempfile + +class TestPreprocessOpenAPI(unittest.TestCase): + def setUp(self): + self.input_file = "tools/tests/generate_proto_py/fixtures/animal.yaml" + self.expected_output_file = "tools/tests/generate_proto_py/fixtures/expected.yaml" + self.temp_dir = tempfile.TemporaryDirectory() + + def test_preprocess_openapi(self): + output_file = os.path.join(self.temp_dir.name, "output.yaml") + + selected_path = "/animals" + + preprocess_openapi(self.input_file, output_file, selected_path) + + with open(output_file, "r") as f: + actual_output = yaml.safe_load(f) + + with open(self.expected_output_file, "r") as f: + expected_output = yaml.safe_load(f) + + self.assertEqual(actual_output, expected_output) + def test_preprocess_openapi_invalid_path(self): + output_file = os.path.join(self.temp_dir.name, "output.yaml") + + invalid_path = "/invalid-path" + + with self.assertRaises(KeyError) as context: + preprocess_openapi(self.input_file, output_file, invalid_path) + + self.assertEqual( + str(context.exception), + f"\"The selected path '{invalid_path}' is not found in the OpenAPI spec.\"" + ) + + def tearDown(self): + self.temp_dir.cleanup() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file