From 01d4643ad6d4aee8c39ed1a1255843baf25b37c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 9 May 2024 16:32:21 +0800 Subject: [PATCH 01/28] =?UTF-8?q?=E6=B7=BB=E5=8A=A0favicon=E5=9B=BE?= =?UTF-8?q?=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/static/img/favicon.ico | Bin 0 -> 3553 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/common/static/img/favicon.ico b/common/static/img/favicon.ico index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5a618b8d56f68f522b70f6c588b9588188931a95 100644 GIT binary patch literal 3553 zcmV<74Ic7|P)(=xOa z5*UhM8p+YFBrCohhmZ_SI)(-c(}CE*p1#MB-VemdJ80Qsq`HBCJ(lP zC$$Wj>0-#Iy{Xi#SNnIawb`gVi`{oA@&BJ^jybvg}nocPmP`}gL))bBFh zIc4$1!q>=zcZpiLSkU^U%ZtFjH&PzAiTcx({ML_|CyZ;?eLOOm_))9 z!tKi*hgFGW#H~Ik)reZVTO_<#$OdzmSUeVCJ{Nz7%S7#VnW?9y`aAO%%uZN0f6j$< z3G-l1w((NpqrtZYOcy*$7yCH|qUir50XC;USnbdedz?DryCCsJxn7=b#KPBy5qrl+Qigf$^rwSOphj(JZoAP_9QO8BhW zJolE`g|p+t;bP*cT1I^jiaJ{TXswBSverQUzRpO#?lzN|ZWHxxLlVi>o5_ZHGc_O= zyHOq~vXFf)H$`lQJBj+b!A$9=B0h@z7v~KB`<44sj7wJfg2azz?H`IkmQEELz_awH zSM&Spjdv28Q^=~svYKy7QN!^_m4)~-nJHp3gp^eu=oMt8OH14qsrc(& zn4?9XGZ}ADaucFj`-j5M(l1GR07utE^Z$bY%<`mDk2j@gmU~jw&7M@vMNgUrjz{W5 z$W4)$A?p$KNmSQnBk@qA;tR-4h-&R0ioqiNld>LgU^@SQZJ=5!4V zgxHX@U^4_GBs~I|V4}WtC#j!}RD6NtH*(ab>{WY>OIB?K0L+@s-w*ujjT-3~zzl&I zz?J~6_oS&8c+=Ipyn+#-3=zx-VZ>-kBF}RP#BikI&p0qA=Q=;joSWzY%t+7LjGxTd z^8hf*n-Tx2H$&6y&Coy;VlzS-Au|C*@oNPB;tTjWx?hrpIQ`u-{Y?AJ;6`s|e6lxF zb5<}y+%v-V29XKetHed@#YgZgvrTNk5U2kwzyxn974OZ|xLPvfVUQ7Hy&;?l^=9fq zy_xzmz#TuwR7iS25%U_r2nfKemP~b4OQz-ru@RA&;5L!pxlNjf0nYFo^O0mAE@ZL+ zhO}hd{>#=(&1qzUV1zg&iA@lX32qbB8ET$@4StqhDl0`l1emflbM9R&S(^4qO^{bd zNu(X%3Q9TB$rw`sFlNIm1nn}bM}ka{rz7cj04H0vm@$S}`Zv-lh(;3+7Vhs z1-JnV%s28JeNnU?^aF&`JSCrmaLWsyzZ{p;k`;e=ygHJN2SKH2V1tMJ9l#BqWq5Hf z7%XHcfC>4NtnIWey?tJ5mgdx?y}&%NsRMAs&(V8jld&fN2INP$u!c%hG*1xE5yZVf zS`ksuWuTq|IKgv_Q#ueEUYO};%Aew@W@|?LGI=iuSrLtSIxSVRNTBUEMKRHH6X-I#2*`;7oDi{j$B` zwpbKjAh4+u!8lN7Am0Z#Fi>E=MYc70Hde(S-&spn?;1}-VGKZLQ2{O>(Isi9MEBEJ z6kh=CF^3w6B?cI&;b@%7zARAM5gTxZrGF8t;v>)ii%wev8L(Jl07NIT0S@pi!ky+K^x2BBN{X3-sMTt3|}Gzp);=e00(%Ed0SSB zE{|pL1)5UTp0I;3>H&<(4RFBE>62vp0&8Pkd;~fMc}W8dszU$=28)>IWj(+i>*6E4 zsniqU4Ip71O%4DY@U!%zvdP%4SQlTI6Evn!{xI_ZSj(us2RLwYy^)X^a4Oct7idhT zEb_%T^g;=Gxugac0}y_e`CfERbQszau8(!`5shXlBXs`9%mAKadc?ipzZ(p<$GZ54 z#>JXv!WuwhWPqQgn(p>E19?axI9Ds86Ealcyi_&T~~FQ3`MY4Z5-(;Nf{*Juoo1yTAVg?Cm`Y zJ3Bj}&*y{t_wR?hcI|@OwrztgEiJHUbsDzfBLeR+i&{TQKgfXUMFc*Z^JgsPxe^1) zGI|FF27LYf{Q>biIy&Lu!#=og->CRow{8uHU%d8R_}PX+WWa@wU%YL~*WOGA=w)(w z>ql??J9R`R!0Dwuo}4aacublXyZi?f1>rPV6|5DM}QN4mN^j+zjHr4e!LI%^z^{?_I7ya&>^^Y z?_RiL$4!iT^oQ0t?up3jkx>H2EuO>KVNQ16?`=aDp$(+9QbH-Q5F^ z9BGFK4+`S%7#F{^Y~#52V{r~(jN26Va3J-JmY#vjM;toUv>iT(h{3@@>SVX?n`6h0 z!>+Dw*w%Ig9z1vu?%A^kZr{Egwzjsy=H})g@z)nwus5K}NZluI{Z1XR1>hupfBy;J z(WA#;S63HoYiolC4jh2HcTX<Ut4)|7sBVKVVc$~Sh}Y&N{gxez@-eh(f>OEV_^@#B3y zq0F49@pnkXud{v{GUXJGrW{yot$<%x*^v`P$^Bz@PT0_C3ZA7`@En~mRLDH#FJ$h4 zIXO4rXn-r@zL~ih=IGJmySux);gKUknK>aos&pC~8)04PCB!c+hs9+Zp}mZSCySEB zYez6_ptGkevk%?M#j|ul;2csBWA+y^&+POC{1 zy{Vn&=ti`!;pdnSyjc&{$!?8z0vs7nImM;pgRQ0921_e8U3Rr|(w1;D8jz7msI{V2 ziw5k`fg71b527p6qG?Lsx>sS9PI^PILr44-;D%(dm55&()l##M)8B+lLe&$hqWmnK zg<38$2~}Bvl*cl|S7O4pHy)E3pe25A=v2$A|8^@0a6w$@SSmKbvW@nri$9gSEzo%- zbmb<`((?nA6vupk(hN#BDD8+3n91W*ffl-qcZ9nm`&8w^xaYx4BMi2s6`NqWtr8pY zBL!VHGQ{cQ1NWj985W?^9q5|ovAeQ8J!=;9RqHhu!kBOp%rvnqex%rSagR8(s>4_s zVBPqIx8jS+y>`2OW~S=XM1dLH=+F^HhmL4Pn=?rKDqS4@H!UhCC@3f>C@3f>C@3f> b037%~u&xOfNpM@h00000NkvXXu0mjfKBBW? literal 0 HcmV?d00001 From ccbb373bd0980e4492a210421fe649c9067d25a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 20 Jun 2024 17:05:35 +0800 Subject: [PATCH 02/28] firset --- common/static/img/favicon.ico | Bin 3553 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/common/static/img/favicon.ico b/common/static/img/favicon.ico index 5a618b8d56f68f522b70f6c588b9588188931a95..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 GIT binary patch literal 0 HcmV?d00001 literal 3553 zcmV<74Ic7|P)(=xOa z5*UhM8p+YFBrCohhmZ_SI)(-c(}CE*p1#MB-VemdJ80Qsq`HBCJ(lP zC$$Wj>0-#Iy{Xi#SNnIawb`gVi`{oA@&BJ^jybvg}nocPmP`}gL))bBFh zIc4$1!q>=zcZpiLSkU^U%ZtFjH&PzAiTcx({ML_|CyZ;?eLOOm_))9 z!tKi*hgFGW#H~Ik)reZVTO_<#$OdzmSUeVCJ{Nz7%S7#VnW?9y`aAO%%uZN0f6j$< z3G-l1w((NpqrtZYOcy*$7yCH|qUir50XC;USnbdedz?DryCCsJxn7=b#KPBy5qrl+Qigf$^rwSOphj(JZoAP_9QO8BhW zJolE`g|p+t;bP*cT1I^jiaJ{TXswBSverQUzRpO#?lzN|ZWHxxLlVi>o5_ZHGc_O= zyHOq~vXFf)H$`lQJBj+b!A$9=B0h@z7v~KB`<44sj7wJfg2azz?H`IkmQEELz_awH zSM&Spjdv28Q^=~svYKy7QN!^_m4)~-nJHp3gp^eu=oMt8OH14qsrc(& zn4?9XGZ}ADaucFj`-j5M(l1GR07utE^Z$bY%<`mDk2j@gmU~jw&7M@vMNgUrjz{W5 z$W4)$A?p$KNmSQnBk@qA;tR-4h-&R0ioqiNld>LgU^@SQZJ=5!4V zgxHX@U^4_GBs~I|V4}WtC#j!}RD6NtH*(ab>{WY>OIB?K0L+@s-w*ujjT-3~zzl&I zz?J~6_oS&8c+=Ipyn+#-3=zx-VZ>-kBF}RP#BikI&p0qA=Q=;joSWzY%t+7LjGxTd z^8hf*n-Tx2H$&6y&Coy;VlzS-Au|C*@oNPB;tTjWx?hrpIQ`u-{Y?AJ;6`s|e6lxF zb5<}y+%v-V29XKetHed@#YgZgvrTNk5U2kwzyxn974OZ|xLPvfVUQ7Hy&;?l^=9fq zy_xzmz#TuwR7iS25%U_r2nfKemP~b4OQz-ru@RA&;5L!pxlNjf0nYFo^O0mAE@ZL+ zhO}hd{>#=(&1qzUV1zg&iA@lX32qbB8ET$@4StqhDl0`l1emflbM9R&S(^4qO^{bd zNu(X%3Q9TB$rw`sFlNIm1nn}bM}ka{rz7cj04H0vm@$S}`Zv-lh(;3+7Vhs z1-JnV%s28JeNnU?^aF&`JSCrmaLWsyzZ{p;k`;e=ygHJN2SKH2V1tMJ9l#BqWq5Hf z7%XHcfC>4NtnIWey?tJ5mgdx?y}&%NsRMAs&(V8jld&fN2INP$u!c%hG*1xE5yZVf zS`ksuWuTq|IKgv_Q#ueEUYO};%Aew@W@|?LGI=iuSrLtSIxSVRNTBUEMKRHH6X-I#2*`;7oDi{j$B` zwpbKjAh4+u!8lN7Am0Z#Fi>E=MYc70Hde(S-&spn?;1}-VGKZLQ2{O>(Isi9MEBEJ z6kh=CF^3w6B?cI&;b@%7zARAM5gTxZrGF8t;v>)ii%wev8L(Jl07NIT0S@pi!ky+K^x2BBN{X3-sMTt3|}Gzp);=e00(%Ed0SSB zE{|pL1)5UTp0I;3>H&<(4RFBE>62vp0&8Pkd;~fMc}W8dszU$=28)>IWj(+i>*6E4 zsniqU4Ip71O%4DY@U!%zvdP%4SQlTI6Evn!{xI_ZSj(us2RLwYy^)X^a4Oct7idhT zEb_%T^g;=Gxugac0}y_e`CfERbQszau8(!`5shXlBXs`9%mAKadc?ipzZ(p<$GZ54 z#>JXv!WuwhWPqQgn(p>E19?axI9Ds86Ealcyi_&T~~FQ3`MY4Z5-(;Nf{*Juoo1yTAVg?Cm`Y zJ3Bj}&*y{t_wR?hcI|@OwrztgEiJHUbsDzfBLeR+i&{TQKgfXUMFc*Z^JgsPxe^1) zGI|FF27LYf{Q>biIy&Lu!#=og->CRow{8uHU%d8R_}PX+WWa@wU%YL~*WOGA=w)(w z>ql??J9R`R!0Dwuo}4aacublXyZi?f1>rPV6|5DM}QN4mN^j+zjHr4e!LI%^z^{?_I7ya&>^^Y z?_RiL$4!iT^oQ0t?up3jkx>H2EuO>KVNQ16?`=aDp$(+9QbH-Q5F^ z9BGFK4+`S%7#F{^Y~#52V{r~(jN26Va3J-JmY#vjM;toUv>iT(h{3@@>SVX?n`6h0 z!>+Dw*w%Ig9z1vu?%A^kZr{Egwzjsy=H})g@z)nwus5K}NZluI{Z1XR1>hupfBy;J z(WA#;S63HoYiolC4jh2HcTX<Ut4)|7sBVKVVc$~Sh}Y&N{gxez@-eh(f>OEV_^@#B3y zq0F49@pnkXud{v{GUXJGrW{yot$<%x*^v`P$^Bz@PT0_C3ZA7`@En~mRLDH#FJ$h4 zIXO4rXn-r@zL~ih=IGJmySux);gKUknK>aos&pC~8)04PCB!c+hs9+Zp}mZSCySEB zYez6_ptGkevk%?M#j|ul;2csBWA+y^&+POC{1 zy{Vn&=ti`!;pdnSyjc&{$!?8z0vs7nImM;pgRQ0921_e8U3Rr|(w1;D8jz7msI{V2 ziw5k`fg71b527p6qG?Lsx>sS9PI^PILr44-;D%(dm55&()l##M)8B+lLe&$hqWmnK zg<38$2~}Bvl*cl|S7O4pHy)E3pe25A=v2$A|8^@0a6w$@SSmKbvW@nri$9gSEzo%- zbmb<`((?nA6vupk(hN#BDD8+3n91W*ffl-qcZ9nm`&8w^xaYx4BMi2s6`NqWtr8pY zBL!VHGQ{cQ1NWj985W?^9q5|ovAeQ8J!=;9RqHhu!kBOp%rvnqex%rSagR8(s>4_s zVBPqIx8jS+y>`2OW~S=XM1dLH=+F^HhmL4Pn=?rKDqS4@H!UhCC@3f>C@3f>C@3f> b037%~u&xOfNpM@h00000NkvXXu0mjfKBBW? From 3b84684023c6400dfb7b91c27c9cf0088dae401d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 20 Jun 2024 17:53:19 +0800 Subject: [PATCH 03/28] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.list b/.env.list index 87f3fc5549..c7bde7b8ba 100644 --- a/.env.list +++ b/.env.list @@ -1,5 +1,5 @@ # https://django-environ.readthedocs.io/en/latest/quickstart.html#usage -# https://docs.djangoproject.com/zh-hans/4.1/ref/settings/ +# https://docs.djangoproject.com/zh-hans/4.1/ref/settings/ DEBUG=false DATABASE_URL=mysql://root:@127.0.0.1:3306/archery CACHE_URL=redis://127.0.0.1:6379/0 From 9f66a6c58daa766c159f928a06d83b590660bb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 20 Jun 2024 17:53:56 +0800 Subject: [PATCH 04/28] =?UTF-8?q?=E6=92=A4=E9=94=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.list b/.env.list index c7bde7b8ba..87f3fc5549 100644 --- a/.env.list +++ b/.env.list @@ -1,5 +1,5 @@ # https://django-environ.readthedocs.io/en/latest/quickstart.html#usage -# https://docs.djangoproject.com/zh-hans/4.1/ref/settings/ +# https://docs.djangoproject.com/zh-hans/4.1/ref/settings/ DEBUG=false DATABASE_URL=mysql://root:@127.0.0.1:3306/archery CACHE_URL=redis://127.0.0.1:6379/0 From d1424f11bbaf115b587cbf5b905a1e2c540808b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Mon, 2 Sep 2024 14:40:50 +0800 Subject: [PATCH 05/28] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BF=BD=E7=95=A5?= =?UTF-8?q?=E8=AF=81=E4=B9=A6=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 2 ++ sql/models.py | 8 ++++++++ src/init_sql/v1.12.0.sql | 3 +++ 3 files changed, 13 insertions(+) create mode 100644 src/init_sql/v1.12.0.sql diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index 6858e6dcfa..b04e1a7fa4 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -27,6 +27,8 @@ def __init__(self, instance=None): self.db_name = instance.db_name self.mode = instance.mode self.is_ssl = instance.is_ssl + self.is_ignore_certificate_error = instance.is_ignore_certificate_error + self.allow_db_name_list = instance.allow_db_name_list # 判断如果配置了隧道则连接隧道，只测试了MySQL if self.instance.tunnel: diff --git a/sql/models.py b/sql/models.py index 460a43e34f..9a7cb2d57b 100755 --- a/sql/models.py +++ b/sql/models.py @@ -205,7 +205,15 @@ class Instance(models.Model): verbose_name="密码", max_length=300, default="", blank=True ) is_ssl = models.BooleanField("是否启用SSL", default=False) + is_ignore_certificate_error = models.BooleanField("是否忽略证书错误", default=False) db_name = models.CharField("数据库", max_length=64, default="", blank=True) + allow_db_name_list = models.CharField( + "允许显示的数据库列表", + max_length=1024, + default="", + blank=True, + help_text="多个以逗号隔开，支持*和~。MySQL示例：test_db,dmp_db,za_ag。Redis示例: 0,4,20~26", + ) charset = models.CharField("字符集", max_length=20, default="", blank=True) service_name = models.CharField( "Oracle service name", max_length=50, null=True, blank=True diff --git a/src/init_sql/v1.12.0.sql b/src/init_sql/v1.12.0.sql new file mode 100644 index 0000000000..a5dddd6dd9 --- /dev/null +++ b/src/init_sql/v1.12.0.sql @@ -0,0 +1,3 @@ + +ALTER TABLE sql_instance ADD is_ignore_certificate_error tinyint(1) DEFAULT 0 COMMENT '是否忽略证书错误。1：忽略。0：不忽略，需要验证'; +ALTER TABLE sql_instance ADD allow_db_name_list varchar(1024) DEFAULT '' COMMENT '允许显示的数据库列表，多个以逗号隔开，支持*和~。'; From 7e7f87d5520ef14299d7f09d78d16ef5f1619007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Mon, 2 Sep 2024 15:03:52 +0800 Subject: [PATCH 06/28] =?UTF-8?q?=E6=94=AF=E6=8C=81-=E5=85=81=E8=AE=B8?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E7=9A=84=E6=95=B0=E6=8D=AE=E5=BA=93=E5=88=97?= =?UTF-8?q?=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mongo.py | 9 +++++-- sql/utils/sql_utils.py | 57 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/sql/engines/mongo.py b/sql/engines/mongo.py index 84581531d9..e45d505d75 100644 --- a/sql/engines/mongo.py +++ b/sql/engines/mongo.py @@ -15,6 +15,8 @@ from bson.objectid import ObjectId from bson.int64 import Int64 +from sql.utils.sql_utils import filter_db_list + from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult from common.config import SysConfig @@ -848,9 +850,12 @@ def get_all_databases(self): result = ResultSet() conn = self.get_connection() try: - result.rows = conn.list_database_names() + db_list = conn.list_database_names() except OperationFailure: - result.rows = [self.db_name] + db_list = [self.db_name] + result.rows = filter_db_list(db_list, self.allow_db_name_list) + return result + return result def get_all_tables(self, db_name, **kwargs): diff --git a/sql/utils/sql_utils.py b/sql/utils/sql_utils.py index d7eb0a19cd..ec065828b2 100644 --- a/sql/utils/sql_utils.py +++ b/sql/utils/sql_utils.py @@ -380,3 +380,60 @@ def get_exec_sqlitem_list(reviewResult, db_name): ) ) return list + + +def filter_db_list(db_list, allow_db_name_list, key="value"): + """ + 根据配置的数据库列表过滤数据库名称列表。 + + :param db_list: 待过滤的数据库名称列表，可能是字符串列表或字典列表。 + 示例数据： + 1. db_list=[{"value": 0, "text": 0, "value": 1, "text": 1}] + 2. db_list=["a_db","b_db"] + :param allow_db_name_list: 配置的数据库显示列表。支持通配符 * 和范围 ~。 + :param key: 当 db_list 包含字典时，指定用于匹配的键。默认值为 'value'。 + :return: 过滤后的数据库名称列表或字典列表。 + """ + if not allow_db_name_list: + return db_list # 如果没有指定 allow_db_name_list，则返回原始 db_list + + # 将通配符模式字符串转换为列表 + allowed_patterns = allow_db_name_list.split(",") + + allowed_regexes = [] + for pattern in allowed_patterns: + pattern = pattern.strip() + if not pattern: + continue + if "~" in pattern: + # 如果模式包含范围 "~"，解析范围并生成数字列表 + try: + start, end = map(int, pattern.split("~")) + # 将生成的数字添加到允许列表的正则表达式列表 + allowed_regexes.extend( + [re.compile(f"^{db}$") for db in range(start, end + 1)] + ) + except ValueError: + # 如果范围内的值不是有效的整数 + allowed_regexes.append(re.compile(f"^{pattern}$".replace("*", ".*"))) + else: + # 转义特殊字符,并将通配符模式转换为正则表达式 + escaped_pattern = re.escape(pattern).replace(r"\*", ".*") + allowed_regexes.append(re.compile(f"^{escaped_pattern}$")) + + filtered_list = [] + + # 根据类型处理 db_list + if all(isinstance(db, dict) for db in db_list): + # 如果 db_list 是一个字典的列表 + for db in db_list: + db_value = db.get(key, "") + if any(regex.match(db_value) for regex in allowed_regexes): + filtered_list.append(db) + else: + # 如果 db_list 是一个字符串的列表 + for db in db_list: + if any(regex.match(db) for regex in allowed_regexes): + filtered_list.append(db) + + return filtered_list From 407e8e4f2d04ad0caad145b9d6de8a9d6408c6e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Mon, 2 Sep 2024 17:38:24 +0800 Subject: [PATCH 07/28] =?UTF-8?q?=E7=AD=9B=E9=80=89=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mongo.py | 2 -- sql/engines/redis.py | 2 ++ sql/engines/tests.py | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sql/engines/mongo.py b/sql/engines/mongo.py index e45d505d75..518e5bd047 100644 --- a/sql/engines/mongo.py +++ b/sql/engines/mongo.py @@ -856,8 +856,6 @@ def get_all_databases(self): result.rows = filter_db_list(db_list, self.allow_db_name_list) return result - return result - def get_all_tables(self, db_name, **kwargs): result = ResultSet() conn = self.get_connection() diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 4df0b258f8..1f84b84580 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -15,6 +15,7 @@ import traceback from common.utils.timer import FuncTimer +from sql.utils.sql_utils import filter_db_list from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult @@ -85,6 +86,7 @@ def get_all_databases(self, **kwargs): rows = max(dbs + [15]) + 1 db_list = [str(x) for x in range(int(rows))] + db_list = filter_db_list(db_list, self.allow_db_name_list) result.rows = db_list return result diff --git a/sql/engines/tests.py b/sql/engines/tests.py index eea86f31c3..14d52da728 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -327,6 +327,17 @@ def test_get_all_databases(self, _config_get): dbs = new_engine.get_all_databases() self.assertListEqual(dbs.rows, ["0", "1", "2", "3"]) + @patch("redis.Redis.config_get", return_value={"databases": 20}) + def test_get_all_databases_with_allow_db_name_list(self, _config_get): + """获取数据库列表。""" + new_engine = RedisEngine(instance=self.ins) + new_engine.allow_db_name_list="0,4,6~9," + dbs = new_engine.get_all_databases() + # 预期结果：过滤后的数据库列表应只包括允许的数据库名 + expected_result = ["0", "4", "6", "7", "8", "9"] + self.assertListEqual(dbs.rows, expected_result) + + @patch("redis.Redis.info") @patch("redis.Redis.config_get") def test_get_all_databases_exception_handling(self, mock_config_get, mock_info): From f9347e8a50739944a456b835a4194a3ec2ee88ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Mon, 2 Sep 2024 17:47:10 +0800 Subject: [PATCH 08/28] =?UTF-8?q?=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95X?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 14d52da728..ff00d10239 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -331,13 +331,12 @@ def test_get_all_databases(self, _config_get): def test_get_all_databases_with_allow_db_name_list(self, _config_get): """获取数据库列表。""" new_engine = RedisEngine(instance=self.ins) - new_engine.allow_db_name_list="0,4,6~9," + new_engine.allow_db_name_list = "0,4,6~9," dbs = new_engine.get_all_databases() # 预期结果：过滤后的数据库列表应只包括允许的数据库名 expected_result = ["0", "4", "6", "7", "8", "9"] self.assertListEqual(dbs.rows, expected_result) - @patch("redis.Redis.info") @patch("redis.Redis.config_get") def test_get_all_databases_exception_handling(self, mock_config_get, mock_info): From de90db473f3828acd3f6efb7ce673d13ce737f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:16:17 +0800 Subject: [PATCH 09/28] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E9=80=89=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 3 +- sql/engines/mongo.py | 6 ++-- sql/engines/mysql.py | 4 ++- sql/engines/redis.py | 5 +-- sql/engines/tests.py | 4 +-- sql/models.py | 14 ++++++-- sql/utils/sql_utils.py | 78 +++++++++++++++++++++++++--------------- src/init_sql/v1.12.0.sql | 4 ++- 8 files changed, 77 insertions(+), 41 deletions(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index b04e1a7fa4..73f6ad3fe8 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -28,7 +28,8 @@ def __init__(self, instance=None): self.mode = instance.mode self.is_ssl = instance.is_ssl self.is_ignore_certificate_error = instance.is_ignore_certificate_error - self.allow_db_name_list = instance.allow_db_name_list + self.show_db_name_regex = instance.show_db_name_regex + self.denied_db_name_regex = instance.denied_db_name_regex # 判断如果配置了隧道则连接隧道，只测试了MySQL if self.instance.tunnel: diff --git a/sql/engines/mongo.py b/sql/engines/mongo.py index 518e5bd047..ee98f27459 100644 --- a/sql/engines/mongo.py +++ b/sql/engines/mongo.py @@ -15,7 +15,7 @@ from bson.objectid import ObjectId from bson.int64 import Int64 -from sql.utils.sql_utils import filter_db_list +from sql.utils.sql_utils import filter_denied_db_list, filter_show_db_list from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult @@ -853,7 +853,9 @@ def get_all_databases(self): db_list = conn.list_database_names() except OperationFailure: db_list = [self.db_name] - result.rows = filter_db_list(db_list, self.allow_db_name_list) + db_list = filter_show_db_list(db_list, self.show_db_name_regex) + db_list = filter_denied_db_list(db_list, self.denied_db_name_regex) + result.rows = db_list return result def get_all_tables(self, db_name, **kwargs): diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index 369931791b..3d558e40e4 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -12,7 +12,7 @@ from schemaobject.connection import build_database_url from sql.engines.goinception import GoInceptionEngine -from sql.utils.sql_utils import get_syntax_type, remove_comments +from sql.utils.sql_utils import filter_denied_db_list, filter_show_db_list, get_syntax_type, remove_comments from . import EngineBase from .models import ResultSet, ReviewResult, ReviewSet from sql.utils.data_masking import data_masking @@ -194,6 +194,8 @@ def get_all_databases(self): db_list = [ row[0] for row in result.rows if row[0] not in self.forbidden_databases ] + db_list = filter_show_db_list(db_list, self.show_db_name_regex) + db_list = filter_denied_db_list(db_list, self.denied_db_name_regex) result.rows = db_list return result diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 1f84b84580..8af138c763 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -15,7 +15,7 @@ import traceback from common.utils.timer import FuncTimer -from sql.utils.sql_utils import filter_db_list +from sql.utils.sql_utils import filter_denied_db_list, filter_show_db_list from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult @@ -86,7 +86,8 @@ def get_all_databases(self, **kwargs): rows = max(dbs + [15]) + 1 db_list = [str(x) for x in range(int(rows))] - db_list = filter_db_list(db_list, self.allow_db_name_list) + db_list = filter_show_db_list(db_list, self.show_db_name_regex) + db_list = filter_denied_db_list(db_list, self.denied_db_name_regex) result.rows = db_list return result diff --git a/sql/engines/tests.py b/sql/engines/tests.py index ff00d10239..20c7cd05ff 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -328,10 +328,10 @@ def test_get_all_databases(self, _config_get): self.assertListEqual(dbs.rows, ["0", "1", "2", "3"]) @patch("redis.Redis.config_get", return_value={"databases": 20}) - def test_get_all_databases_with_allow_db_name_list(self, _config_get): + def test_get_all_databases_with_show_db_name_regex(self, _config_get): """获取数据库列表。""" new_engine = RedisEngine(instance=self.ins) - new_engine.allow_db_name_list = "0,4,6~9," + new_engine.show_db_name_regex = "0,4,6~9," dbs = new_engine.get_all_databases() # 预期结果：过滤后的数据库列表应只包括允许的数据库名 expected_result = ["0", "4", "6", "7", "8", "9"] diff --git a/sql/models.py b/sql/models.py index 9a7cb2d57b..063a2058a5 100755 --- a/sql/models.py +++ b/sql/models.py @@ -207,13 +207,21 @@ class Instance(models.Model): is_ssl = models.BooleanField("是否启用SSL", default=False) is_ignore_certificate_error = models.BooleanField("是否忽略证书错误", default=False) db_name = models.CharField("数据库", max_length=64, default="", blank=True) - allow_db_name_list = models.CharField( - "允许显示的数据库列表", + show_db_name_regex = models.CharField( + "显示的数据库列表正则", max_length=1024, default="", blank=True, - help_text="多个以逗号隔开，支持*和~。MySQL示例：test_db,dmp_db,za_ag。Redis示例: 0,4,20~26", + help_text="正则表达式。示例：^(test_db|dmp_db|za.*)$。Redis示例: ^(0|4|6|11|12|13)$", ) + denied_db_name_regex = models.CharField( + "隐藏的数据库列表正则", + max_length=1024, + default="", + blank=True, + help_text="正则表达式。隐藏大于显示，此规则优先。", + ) + charset = models.CharField("字符集", max_length=20, default="", blank=True) service_name = models.CharField( "Oracle service name", max_length=50, null=True, blank=True diff --git a/sql/utils/sql_utils.py b/sql/utils/sql_utils.py index ec065828b2..acf0935a94 100644 --- a/sql/utils/sql_utils.py +++ b/sql/utils/sql_utils.py @@ -382,7 +382,7 @@ def get_exec_sqlitem_list(reviewResult, db_name): return list -def filter_db_list(db_list, allow_db_name_list, key="value"): +def filter_show_db_list(db_list, show_db_name_regex, key="value"): """ 根据配置的数据库列表过滤数据库名称列表。 @@ -390,36 +390,54 @@ def filter_db_list(db_list, allow_db_name_list, key="value"): 示例数据： 1. db_list=[{"value": 0, "text": 0, "value": 1, "text": 1}] 2. db_list=["a_db","b_db"] - :param allow_db_name_list: 配置的数据库显示列表。支持通配符 * 和范围 ~。 + :param show_db_name_regex: 配置的数据库显示正则。 :param key: 当 db_list 包含字典时，指定用于匹配的键。默认值为 'value'。 :return: 过滤后的数据库名称列表或字典列表。 """ - if not allow_db_name_list: - return db_list # 如果没有指定 allow_db_name_list，则返回原始 db_list + if not show_db_name_regex: + return db_list # 如果没有指定 show_db_name_regex，返回原始 db_list - # 将通配符模式字符串转换为列表 - allowed_patterns = allow_db_name_list.split(",") + try: + allowed_regex = re.compile(show_db_name_regex) # 编译正则表达式 + except re.error: + raise ValueError(f"正则表达式解析异常: {show_db_name_regex}") - allowed_regexes = [] - for pattern in allowed_patterns: - pattern = pattern.strip() - if not pattern: - continue - if "~" in pattern: - # 如果模式包含范围 "~"，解析范围并生成数字列表 - try: - start, end = map(int, pattern.split("~")) - # 将生成的数字添加到允许列表的正则表达式列表 - allowed_regexes.extend( - [re.compile(f"^{db}$") for db in range(start, end + 1)] - ) - except ValueError: - # 如果范围内的值不是有效的整数 - allowed_regexes.append(re.compile(f"^{pattern}$".replace("*", ".*"))) - else: - # 转义特殊字符,并将通配符模式转换为正则表达式 - escaped_pattern = re.escape(pattern).replace(r"\*", ".*") - allowed_regexes.append(re.compile(f"^{escaped_pattern}$")) + filtered_list = [] + + # 根据类型处理 db_list + if all(isinstance(db, dict) for db in db_list): + # 如果 db_list 是一个字典的列表 + for db in db_list: + db_value = str(db.get(key, "")) # 确保 db_value 是字符串类型 + if allowed_regex.match(db_value): + filtered_list.append(db) + else: + # 如果 db_list 是一个字符串的列表 + for db in db_list: + if allowed_regex.match(db): + filtered_list.append(db) + + return filtered_list + + +def filter_denied_db_list(db_list, denied_db_name_regex, key="value"): + """ + 根据配置的数据库列表过滤数据库名称列表。 + + :param db_list: 待过滤的数据库名称列表，可能是字符串列表或字典列表。 + 示例数据： + 1. db_list=[{"value": 0, "text": 0, "value": 1, "text": 1}] + 2. db_list=["a_db","b_db"] + :param denied_db_name_regex: 配置的数据库隐藏正则。 + :param key: 当 db_list 包含字典时，指定用于匹配的键。默认值为 'value'。 + :return: 过滤后的数据库名称列表或字典列表。 + """ + if not denied_db_name_regex: + return db_list + try: + denied_regex = re.compile(denied_db_name_regex) + except re.error: + raise ValueError(f"正则表达式解析异常: {denied_db_name_regex}") filtered_list = [] @@ -427,13 +445,15 @@ def filter_db_list(db_list, allow_db_name_list, key="value"): if all(isinstance(db, dict) for db in db_list): # 如果 db_list 是一个字典的列表 for db in db_list: - db_value = db.get(key, "") - if any(regex.match(db_value) for regex in allowed_regexes): + db_value = str(db.get(key, "")) # 确保 db_value 是字符串类型 + if not denied_regex.match( + db_value + ): # 只有不匹配隐藏正则表达式的才添加到列表中 filtered_list.append(db) else: # 如果 db_list 是一个字符串的列表 for db in db_list: - if any(regex.match(db) for regex in allowed_regexes): + if not denied_regex.match(db): # 只有不匹配隐藏正则表达式的才添加到列表中 filtered_list.append(db) return filtered_list diff --git a/src/init_sql/v1.12.0.sql b/src/init_sql/v1.12.0.sql index a5dddd6dd9..37690e18c1 100644 --- a/src/init_sql/v1.12.0.sql +++ b/src/init_sql/v1.12.0.sql @@ -1,3 +1,5 @@ ALTER TABLE sql_instance ADD is_ignore_certificate_error tinyint(1) DEFAULT 0 COMMENT '是否忽略证书错误。1：忽略。0：不忽略，需要验证'; -ALTER TABLE sql_instance ADD allow_db_name_list varchar(1024) DEFAULT '' COMMENT '允许显示的数据库列表，多个以逗号隔开，支持*和~。'; +ALTER TABLE sql_instance ADD show_db_name_regex varchar(1024) DEFAULT '' COMMENT '显示的数据库列表正则'; +ALTER TABLE sql_instance ADD denied_db_name_regex varchar(1024) DEFAULT '' COMMENT '隐藏的数据库列表正则'; + \ No newline at end of file From 219bac4497303f6b593e1a8abd44853cf91476d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:19:01 +0800 Subject: [PATCH 10/28] show_db_name_regex --- sql/engines/tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 20c7cd05ff..ccc762cc19 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -331,10 +331,11 @@ def test_get_all_databases(self, _config_get): def test_get_all_databases_with_show_db_name_regex(self, _config_get): """获取数据库列表。""" new_engine = RedisEngine(instance=self.ins) - new_engine.show_db_name_regex = "0,4,6~9," + new_engine.show_db_name_regex = "^(0|4|6|11|12|13)$" + new_engine.denied_db_name_regex = "^(4|13|22)$" dbs = new_engine.get_all_databases() # 预期结果：过滤后的数据库列表应只包括允许的数据库名 - expected_result = ["0", "4", "6", "7", "8", "9"] + expected_result = ["0", "6", "11", "12", "13"] self.assertListEqual(dbs.rows, expected_result) @patch("redis.Redis.info") From 9a29c48d2d957193eea7b8e428cd9e5fea421860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:21:00 +0800 Subject: [PATCH 11/28] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mysql.py | 7 ++++++- sql/engines/tests.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index 3d558e40e4..5ca2d8fcf8 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -12,7 +12,12 @@ from schemaobject.connection import build_database_url from sql.engines.goinception import GoInceptionEngine -from sql.utils.sql_utils import filter_denied_db_list, filter_show_db_list, get_syntax_type, remove_comments +from sql.utils.sql_utils import ( + filter_denied_db_list, + filter_show_db_list, + get_syntax_type, + remove_comments, +) from . import EngineBase from .models import ResultSet, ReviewResult, ReviewSet from sql.utils.data_masking import data_masking diff --git a/sql/engines/tests.py b/sql/engines/tests.py index ccc762cc19..4684a9f82a 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -335,7 +335,7 @@ def test_get_all_databases_with_show_db_name_regex(self, _config_get): new_engine.denied_db_name_regex = "^(4|13|22)$" dbs = new_engine.get_all_databases() # 预期结果：过滤后的数据库列表应只包括允许的数据库名 - expected_result = ["0", "6", "11", "12", "13"] + expected_result = ["0", "6", "11", "12"] self.assertListEqual(dbs.rows, expected_result) @patch("redis.Redis.info") From 6c9eeca0701f10e747d7392cd0a24597041efc6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:28:06 +0800 Subject: [PATCH 12/28] =?UTF-8?q?=E6=98=AF=E5=90=A6=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AFSSL=E8=AF=81=E4=B9=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/models.py b/sql/models.py index 063a2058a5..d86a985730 100755 --- a/sql/models.py +++ b/sql/models.py @@ -205,7 +205,7 @@ class Instance(models.Model): verbose_name="密码", max_length=300, default="", blank=True ) is_ssl = models.BooleanField("是否启用SSL", default=False) - is_ignore_certificate_error = models.BooleanField("是否忽略证书错误", default=False) + verify_ssl= models.BooleanField("是否验证服务端SSL证书", default=True) db_name = models.CharField("数据库", max_length=64, default="", blank=True) show_db_name_regex = models.CharField( "显示的数据库列表正则", From 9945b2497824ec0d80dc824207e0fff4e2c81f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:30:58 +0800 Subject: [PATCH 13/28] =?UTF-8?q?=E6=98=AF=E5=90=A6=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AFSSL=E8=AF=81=E4=B9=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 2 +- src/init_sql/v1.12.0.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index 73f6ad3fe8..d624ad5aa1 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -27,7 +27,7 @@ def __init__(self, instance=None): self.db_name = instance.db_name self.mode = instance.mode self.is_ssl = instance.is_ssl - self.is_ignore_certificate_error = instance.is_ignore_certificate_error + self.verify_ssl = instance.verify_ssl self.show_db_name_regex = instance.show_db_name_regex self.denied_db_name_regex = instance.denied_db_name_regex diff --git a/src/init_sql/v1.12.0.sql b/src/init_sql/v1.12.0.sql index 37690e18c1..22d4fbbde7 100644 --- a/src/init_sql/v1.12.0.sql +++ b/src/init_sql/v1.12.0.sql @@ -1,5 +1,5 @@ -ALTER TABLE sql_instance ADD is_ignore_certificate_error tinyint(1) DEFAULT 0 COMMENT '是否忽略证书错误。1：忽略。0：不忽略，需要验证'; +ALTER TABLE sql_instance ADD verify_ssl tinyint(1) DEFAULT 1 COMMENT '是否验证服务端SSL证书。1：验证。0：不验证'; ALTER TABLE sql_instance ADD show_db_name_regex varchar(1024) DEFAULT '' COMMENT '显示的数据库列表正则'; ALTER TABLE sql_instance ADD denied_db_name_regex varchar(1024) DEFAULT '' COMMENT '隐藏的数据库列表正则'; \ No newline at end of file From 0e877717ad0e134d8247b652d32264b6e70d94f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:31:20 +0800 Subject: [PATCH 14/28] =?UTF-8?q?=E6=98=AF=E5=90=A6=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AFSSL=E8=AF=81=E4=B9=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/init_sql/v1.12.0.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init_sql/v1.12.0.sql b/src/init_sql/v1.12.0.sql index 22d4fbbde7..799e130e4c 100644 --- a/src/init_sql/v1.12.0.sql +++ b/src/init_sql/v1.12.0.sql @@ -1,4 +1,4 @@ - +-- 2024-9 ALTER TABLE sql_instance ADD verify_ssl tinyint(1) DEFAULT 1 COMMENT '是否验证服务端SSL证书。1：验证。0：不验证'; ALTER TABLE sql_instance ADD show_db_name_regex varchar(1024) DEFAULT '' COMMENT '显示的数据库列表正则'; ALTER TABLE sql_instance ADD denied_db_name_regex varchar(1024) DEFAULT '' COMMENT '隐藏的数据库列表正则'; From ab8d9405cfccf7d61cf80f9fec10478b993b28b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:32:30 +0800 Subject: [PATCH 15/28] =?UTF-8?q?=E6=98=AF=E5=90=A6=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E7=AB=AFSSL=E8=AF=81=E4=B9=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/models.py b/sql/models.py index d86a985730..942b2fdb15 100755 --- a/sql/models.py +++ b/sql/models.py @@ -205,7 +205,7 @@ class Instance(models.Model): verbose_name="密码", max_length=300, default="", blank=True ) is_ssl = models.BooleanField("是否启用SSL", default=False) - verify_ssl= models.BooleanField("是否验证服务端SSL证书", default=True) + verify_ssl = models.BooleanField("是否验证服务端SSL证书", default=True) db_name = models.CharField("数据库", max_length=64, default="", blank=True) show_db_name_regex = models.CharField( "显示的数据库列表正则", From 7ddab99e44db1c59f85ee91731364c5e464a970f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 11:45:05 +0800 Subject: [PATCH 16/28] =?UTF-8?q?=E4=BD=BF=E7=94=A8self.instance=E5=B1=9E?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 6 ++---- sql/engines/mongo.py | 4 ++-- sql/engines/mysql.py | 4 ++-- sql/engines/redis.py | 4 ++-- sql/engines/tests.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index d624ad5aa1..a8374f0c4d 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -2,6 +2,7 @@ import importlib from sql.engines.models import ResultSet, ReviewSet +from sql.models import Instance from sql.utils.ssh_tunnel import SSHConnection from django.conf import settings @@ -18,7 +19,7 @@ def __init__(self, instance=None): self.conn = None self.thread_id = None if instance: - self.instance = instance + self.instance = instance # type: Instance self.instance_name = instance.instance_name self.host = instance.host self.port = int(instance.port) @@ -27,9 +28,6 @@ def __init__(self, instance=None): self.db_name = instance.db_name self.mode = instance.mode self.is_ssl = instance.is_ssl - self.verify_ssl = instance.verify_ssl - self.show_db_name_regex = instance.show_db_name_regex - self.denied_db_name_regex = instance.denied_db_name_regex # 判断如果配置了隧道则连接隧道，只测试了MySQL if self.instance.tunnel: diff --git a/sql/engines/mongo.py b/sql/engines/mongo.py index ee98f27459..dc03fa6590 100644 --- a/sql/engines/mongo.py +++ b/sql/engines/mongo.py @@ -853,8 +853,8 @@ def get_all_databases(self): db_list = conn.list_database_names() except OperationFailure: db_list = [self.db_name] - db_list = filter_show_db_list(db_list, self.show_db_name_regex) - db_list = filter_denied_db_list(db_list, self.denied_db_name_regex) + db_list = filter_show_db_list(db_list, self.instance.show_db_name_regex) + db_list = filter_denied_db_list(db_list, self.instance.denied_db_name_regex) result.rows = db_list return result diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index 5ca2d8fcf8..1869b76207 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -199,8 +199,8 @@ def get_all_databases(self): db_list = [ row[0] for row in result.rows if row[0] not in self.forbidden_databases ] - db_list = filter_show_db_list(db_list, self.show_db_name_regex) - db_list = filter_denied_db_list(db_list, self.denied_db_name_regex) + db_list = filter_show_db_list(db_list, self.instance.show_db_name_regex) + db_list = filter_denied_db_list(db_list, self.instance.denied_db_name_regex) result.rows = db_list return result diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 8af138c763..349b8dbe13 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -86,8 +86,8 @@ def get_all_databases(self, **kwargs): rows = max(dbs + [15]) + 1 db_list = [str(x) for x in range(int(rows))] - db_list = filter_show_db_list(db_list, self.show_db_name_regex) - db_list = filter_denied_db_list(db_list, self.denied_db_name_regex) + db_list = filter_show_db_list(db_list, self.instance.show_db_name_regex) + db_list = filter_denied_db_list(db_list, self.instance.denied_db_name_regex) result.rows = db_list return result diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 4684a9f82a..3db7216311 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -331,7 +331,7 @@ def test_get_all_databases(self, _config_get): def test_get_all_databases_with_show_db_name_regex(self, _config_get): """获取数据库列表。""" new_engine = RedisEngine(instance=self.ins) - new_engine.show_db_name_regex = "^(0|4|6|11|12|13)$" + self.ins.show_db_name_regex = "^(0|4|6|11|12|13)$" new_engine.denied_db_name_regex = "^(4|13|22)$" dbs = new_engine.get_all_databases() # 预期结果：过滤后的数据库列表应只包括允许的数据库名 From 0828a9c28263e31b4d30b8273dadaf7b93276c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 12:00:35 +0800 Subject: [PATCH 17/28] =?UTF-8?q?=E4=BD=BF=E7=94=A8ins=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 2 +- sql/engines/tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index a8374f0c4d..8038e24c47 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -15,7 +15,7 @@ class EngineBase: name = "Base" info = "base engine" - def __init__(self, instance=None): + def __init__(self, instance: Instance=None): self.conn = None self.thread_id = None if instance: diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 3db7216311..5162c353be 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -332,7 +332,7 @@ def test_get_all_databases_with_show_db_name_regex(self, _config_get): """获取数据库列表。""" new_engine = RedisEngine(instance=self.ins) self.ins.show_db_name_regex = "^(0|4|6|11|12|13)$" - new_engine.denied_db_name_regex = "^(4|13|22)$" + self.ins.denied_db_name_regex = "^(4|13|22)$" dbs = new_engine.get_all_databases() # 预期结果：过滤后的数据库列表应只包括允许的数据库名 expected_result = ["0", "6", "11", "12"] From 99daa5f8424a0a606948161a7146f63cf5716089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 12:04:17 +0800 Subject: [PATCH 18/28] =?UTF-8?q?=E4=BD=BF=E7=94=A8Instance=E5=B1=9E?= =?UTF-8?q?=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index 8038e24c47..cfcace7acd 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -15,7 +15,7 @@ class EngineBase: name = "Base" info = "base engine" - def __init__(self, instance: Instance=None): + def __init__(self, instance: Instance = None): self.conn = None self.thread_id = None if instance: From 3753992913537c2d2a73a082e1e9898c136dfb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 5 Sep 2024 13:55:19 +0800 Subject: [PATCH 19/28] =?UTF-8?q?=E5=90=88=E5=B9=B6=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E8=BF=87=E6=BB=A4=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mongo.py | 6 ++-- sql/engines/mysql.py | 7 ++--- sql/engines/redis.py | 6 ++-- sql/utils/sql_utils.py | 71 +++++++++--------------------------------- 4 files changed, 23 insertions(+), 67 deletions(-) diff --git a/sql/engines/mongo.py b/sql/engines/mongo.py index dc03fa6590..aca5300b60 100644 --- a/sql/engines/mongo.py +++ b/sql/engines/mongo.py @@ -15,7 +15,7 @@ from bson.objectid import ObjectId from bson.int64 import Int64 -from sql.utils.sql_utils import filter_denied_db_list, filter_show_db_list +from sql.utils.sql_utils import filter_db_list from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult @@ -853,8 +853,8 @@ def get_all_databases(self): db_list = conn.list_database_names() except OperationFailure: db_list = [self.db_name] - db_list = filter_show_db_list(db_list, self.instance.show_db_name_regex) - db_list = filter_denied_db_list(db_list, self.instance.denied_db_name_regex) + db_list = filter_db_list(db_list, self.instance.show_db_name_regex, True) + db_list = filter_db_list(db_list, self.instance.denied_db_name_regex, False) result.rows = db_list return result diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index 1869b76207..3c71080a96 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -13,8 +13,7 @@ from sql.engines.goinception import GoInceptionEngine from sql.utils.sql_utils import ( - filter_denied_db_list, - filter_show_db_list, + filter_db_list, get_syntax_type, remove_comments, ) @@ -199,8 +198,8 @@ def get_all_databases(self): db_list = [ row[0] for row in result.rows if row[0] not in self.forbidden_databases ] - db_list = filter_show_db_list(db_list, self.instance.show_db_name_regex) - db_list = filter_denied_db_list(db_list, self.instance.denied_db_name_regex) + db_list = filter_db_list(db_list, self.instance.show_db_name_regex, True) + db_list = filter_db_list(db_list, self.instance.denied_db_name_regex, False) result.rows = db_list return result diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 349b8dbe13..00543d7306 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -15,7 +15,7 @@ import traceback from common.utils.timer import FuncTimer -from sql.utils.sql_utils import filter_denied_db_list, filter_show_db_list +from sql.utils.sql_utils import filter_db_list from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult @@ -86,8 +86,8 @@ def get_all_databases(self, **kwargs): rows = max(dbs + [15]) + 1 db_list = [str(x) for x in range(int(rows))] - db_list = filter_show_db_list(db_list, self.instance.show_db_name_regex) - db_list = filter_denied_db_list(db_list, self.instance.denied_db_name_regex) + db_list = filter_db_list(db_list, self.instance.show_db_name_regex, True) + db_list = filter_db_list(db_list, self.instance.denied_db_name_regex, False) result.rows = db_list return result diff --git a/sql/utils/sql_utils.py b/sql/utils/sql_utils.py index acf0935a94..36eed15f36 100644 --- a/sql/utils/sql_utils.py +++ b/sql/utils/sql_utils.py @@ -382,7 +382,7 @@ def get_exec_sqlitem_list(reviewResult, db_name): return list -def filter_show_db_list(db_list, show_db_name_regex, key="value"): +def filter_db_list(db_list, db_name_regex: str, is_match_regex: bool, key="value"): """ 根据配置的数据库列表过滤数据库名称列表。 @@ -390,70 +390,27 @@ def filter_show_db_list(db_list, show_db_name_regex, key="value"): 示例数据： 1. db_list=[{"value": 0, "text": 0, "value": 1, "text": 1}] 2. db_list=["a_db","b_db"] - :param show_db_name_regex: 配置的数据库显示正则。 + :param db_name_regex: 配置的数据库正则。 + :param is_match_regex 是匹配还是不匹配的正则 :param key: 当 db_list 包含字典时，指定用于匹配的键。默认值为 'value'。 :return: 过滤后的数据库名称列表或字典列表。 """ - if not show_db_name_regex: - return db_list # 如果没有指定 show_db_name_regex，返回原始 db_list + if not db_name_regex: + return db_list # 如果没有指定 db_name_regex，返回原始 db_list try: - allowed_regex = re.compile(show_db_name_regex) # 编译正则表达式 + db_regex = re.compile(db_name_regex) # 编译正则表达式 except re.error: - raise ValueError(f"正则表达式解析异常: {show_db_name_regex}") + raise ValueError(f"正则表达式解析异常: {db_name_regex}") filtered_list = [] # 根据类型处理 db_list - if all(isinstance(db, dict) for db in db_list): - # 如果 db_list 是一个字典的列表 - for db in db_list: - db_value = str(db.get(key, "")) # 确保 db_value 是字符串类型 - if allowed_regex.match(db_value): - filtered_list.append(db) - else: - # 如果 db_list 是一个字符串的列表 - for db in db_list: - if allowed_regex.match(db): - filtered_list.append(db) - - return filtered_list - - -def filter_denied_db_list(db_list, denied_db_name_regex, key="value"): - """ - 根据配置的数据库列表过滤数据库名称列表。 - - :param db_list: 待过滤的数据库名称列表，可能是字符串列表或字典列表。 - 示例数据： - 1. db_list=[{"value": 0, "text": 0, "value": 1, "text": 1}] - 2. db_list=["a_db","b_db"] - :param denied_db_name_regex: 配置的数据库隐藏正则。 - :param key: 当 db_list 包含字典时，指定用于匹配的键。默认值为 'value'。 - :return: 过滤后的数据库名称列表或字典列表。 - """ - if not denied_db_name_regex: - return db_list - try: - denied_regex = re.compile(denied_db_name_regex) - except re.error: - raise ValueError(f"正则表达式解析异常: {denied_db_name_regex}") - - filtered_list = [] - - # 根据类型处理 db_list - if all(isinstance(db, dict) for db in db_list): - # 如果 db_list 是一个字典的列表 - for db in db_list: - db_value = str(db.get(key, "")) # 确保 db_value 是字符串类型 - if not denied_regex.match( - db_value - ): # 只有不匹配隐藏正则表达式的才添加到列表中 - filtered_list.append(db) - else: - # 如果 db_list 是一个字符串的列表 - for db in db_list: - if not denied_regex.match(db): # 只有不匹配隐藏正则表达式的才添加到列表中 - filtered_list.append(db) - + for db in db_list: + # 确定要检查的值（字符串或字典中的特定键） + db_value = str(db[key]) if isinstance(db, dict) else db + is_match = bool(db_regex.match(db_value)) + # 根据 is_match_regex 参数过滤 + if (is_match_regex and is_match) or (not is_match_regex and not is_match): + filtered_list.append(db) return filtered_list From e05ec6852e0aaf543c5c364a1f1425f962118c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 15:43:10 +0800 Subject: [PATCH 20/28] =?UTF-8?q?filter=5Fdb=5Flist=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E7=BB=9F=E4=B8=80=E5=87=BA=E5=8F=A3=E7=9A=84=E5=9C=B0?= =?UTF-8?q?=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mongo.py | 4 ---- sql/engines/redis.py | 3 --- sql/engines/tests.py | 11 ----------- 3 files changed, 18 deletions(-) diff --git a/sql/engines/mongo.py b/sql/engines/mongo.py index aca5300b60..a989067941 100644 --- a/sql/engines/mongo.py +++ b/sql/engines/mongo.py @@ -15,8 +15,6 @@ from bson.objectid import ObjectId from bson.int64 import Int64 -from sql.utils.sql_utils import filter_db_list - from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult from common.config import SysConfig @@ -853,8 +851,6 @@ def get_all_databases(self): db_list = conn.list_database_names() except OperationFailure: db_list = [self.db_name] - db_list = filter_db_list(db_list, self.instance.show_db_name_regex, True) - db_list = filter_db_list(db_list, self.instance.denied_db_name_regex, False) result.rows = db_list return result diff --git a/sql/engines/redis.py b/sql/engines/redis.py index 00543d7306..4df0b258f8 100644 --- a/sql/engines/redis.py +++ b/sql/engines/redis.py @@ -15,7 +15,6 @@ import traceback from common.utils.timer import FuncTimer -from sql.utils.sql_utils import filter_db_list from . import EngineBase from .models import ResultSet, ReviewSet, ReviewResult @@ -86,8 +85,6 @@ def get_all_databases(self, **kwargs): rows = max(dbs + [15]) + 1 db_list = [str(x) for x in range(int(rows))] - db_list = filter_db_list(db_list, self.instance.show_db_name_regex, True) - db_list = filter_db_list(db_list, self.instance.denied_db_name_regex, False) result.rows = db_list return result diff --git a/sql/engines/tests.py b/sql/engines/tests.py index 5162c353be..eea86f31c3 100644 --- a/sql/engines/tests.py +++ b/sql/engines/tests.py @@ -327,17 +327,6 @@ def test_get_all_databases(self, _config_get): dbs = new_engine.get_all_databases() self.assertListEqual(dbs.rows, ["0", "1", "2", "3"]) - @patch("redis.Redis.config_get", return_value={"databases": 20}) - def test_get_all_databases_with_show_db_name_regex(self, _config_get): - """获取数据库列表。""" - new_engine = RedisEngine(instance=self.ins) - self.ins.show_db_name_regex = "^(0|4|6|11|12|13)$" - self.ins.denied_db_name_regex = "^(4|13|22)$" - dbs = new_engine.get_all_databases() - # 预期结果：过滤后的数据库列表应只包括允许的数据库名 - expected_result = ["0", "6", "11", "12"] - self.assertListEqual(dbs.rows, expected_result) - @patch("redis.Redis.info") @patch("redis.Redis.config_get") def test_get_all_databases_exception_handling(self, mock_config_get, mock_info): From 9e3168ded63de0b9ec59d018ff10a1a9c35db557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 15:43:47 +0800 Subject: [PATCH 21/28] =?UTF-8?q?=E8=BF=87=E6=BB=A4=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E6=94=B9=E4=B8=BA=E7=BB=9F=E4=B8=80=E5=87=BA=E5=8F=A3?= =?UTF-8?q?=E5=9C=B0=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mysql.py | 2 -- sql/instance.py | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index 3c71080a96..a25422fb09 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -198,8 +198,6 @@ def get_all_databases(self): db_list = [ row[0] for row in result.rows if row[0] not in self.forbidden_databases ] - db_list = filter_db_list(db_list, self.instance.show_db_name_regex, True) - db_list = filter_db_list(db_list, self.instance.denied_db_name_regex, False) result.rows = db_list return result diff --git a/sql/instance.py b/sql/instance.py index 54486be745..a9b9332e2e 100644 --- a/sql/instance.py +++ b/sql/instance.py @@ -14,6 +14,7 @@ from common.utils.convert import Convert from sql.engines import get_engine from sql.plugins.schemasync import SchemaSync +from sql.utils.sql_utils import filter_db_list from .models import Instance, ParamTemplate, ParamHistory @@ -332,6 +333,12 @@ def instance_resource(request): tb_name = query_engine.escape_string(tb_name) if resource_type == "database": resource = query_engine.get_all_databases() + resource = filter_db_list( + resource, query_engine.instance.show_db_name_regex, True + ) + resource = filter_db_list( + resource, query_engine.instance.denied_db_name_regex, False + ) elif resource_type == "schema" and db_name: resource = query_engine.get_all_schemas(db_name=db_name) elif resource_type == "table" and db_name: From 1d47729626ef59bc511294c9443c8dcb1c4dfda5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 15:45:06 +0800 Subject: [PATCH 22/28] =?UTF-8?q?=E4=BF=AE=E6=94=B9filter=5Fdb=5Flist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql_api/api_instance.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql_api/api_instance.py b/sql_api/api_instance.py index d25f93713e..1e3b3931a1 100644 --- a/sql_api/api_instance.py +++ b/sql_api/api_instance.py @@ -1,6 +1,8 @@ from rest_framework import views, generics, status, serializers from rest_framework.response import Response from drf_spectacular.utils import extend_schema + +from sql.utils.sql_utils import filter_db_list from .serializers import ( InstanceSerializer, InstanceDetailSerializer, @@ -193,6 +195,12 @@ def post(self, request): tb_name = query_engine.escape_string(tb_name) if resource_type == "database": resource = query_engine.get_all_databases() + resource = filter_db_list( + resource, query_engine.instance.show_db_name_regex, True + ) + resource = filter_db_list( + resource, query_engine.instance.denied_db_name_regex, False + ) elif resource_type == "schema" and db_name: resource = query_engine.get_all_schemas(db_name=db_name) elif resource_type == "table" and db_name: From 7f98347b6bc5132f07ab9cd00ced42e0c27c4576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 17:03:26 +0800 Subject: [PATCH 23/28] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/__init__.py | 1 + sql/engines/mysql.py | 1 - sql/utils/sql_utils.py | 2 + sql/utils/test_sql_utils.py | 454 ++++++++++++++++++++++++++++++++++++ sql/utils/tests.py | 341 --------------------------- sql_api/api_instance.py | 1 + 6 files changed, 458 insertions(+), 342 deletions(-) create mode 100644 sql/utils/test_sql_utils.py diff --git a/sql/engines/__init__.py b/sql/engines/__init__.py index cfcace7acd..92b9b2a8b4 100644 --- a/sql/engines/__init__.py +++ b/sql/engines/__init__.py @@ -1,6 +1,7 @@ """engine base库, 包含一个``EngineBase`` class和一个get_engine函数""" import importlib +import re from sql.engines.models import ResultSet, ReviewSet from sql.models import Instance from sql.utils.ssh_tunnel import SSHConnection diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index a25422fb09..c34b865bc8 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -13,7 +13,6 @@ from sql.engines.goinception import GoInceptionEngine from sql.utils.sql_utils import ( - filter_db_list, get_syntax_type, remove_comments, ) diff --git a/sql/utils/sql_utils.py b/sql/utils/sql_utils.py index 36eed15f36..8cf30df0f3 100644 --- a/sql/utils/sql_utils.py +++ b/sql/utils/sql_utils.py @@ -414,3 +414,5 @@ def filter_db_list(db_list, db_name_regex: str, is_match_regex: bool, key="value if (is_match_regex and is_match) or (not is_match_regex and not is_match): filtered_list.append(db) return filtered_list + + diff --git a/sql/utils/test_sql_utils.py b/sql/utils/test_sql_utils.py new file mode 100644 index 0000000000..d75c680d67 --- /dev/null +++ b/sql/utils/test_sql_utils.py @@ -0,0 +1,454 @@ +# -*- coding: UTF-8 -*- +""" +@author: hhyo +@license: Apache Licence +@file: tests.py +@time: 2019/03/14 + +""" + +from unittest.mock import patch, MagicMock +from django.test import TestCase, Client +from sql.utils.sql_utils import * + + +__author__ = "hhyo" + + +class TestSQLUtils(TestCase): + def test_get_syntax_type(self): + """ + 测试语法判断 + :return: + """ + dml_sql = "select * from users;" + ddl_sql = "alter table users add id not null default 0 comment 'id' " + self.assertEqual(get_syntax_type(dml_sql), "DML") + self.assertEqual(get_syntax_type(ddl_sql), "DDL") + + def test_get_syntax_type_by_re(self): + """ + 测试语法判断，不使用sqlparse解析,直接正则匹配判断 + :return: + """ + dml_sql = "select * from users;" + ddl_sql = "alter table users add id int not null default 0 comment 'id' " + other_sql = "show engine innodb status" + self.assertEqual(get_syntax_type(dml_sql, parser=False, db_type="mysql"), "DML") + self.assertEqual(get_syntax_type(ddl_sql, parser=False, db_type="mysql"), "DDL") + self.assertIsNone(get_syntax_type(other_sql, parser=False, db_type="mysql")) + + def test_remove_comments(self): + """ + 测试去除SQL注释 + :return: + """ + sql1 = """ # This comment continues to the end of line + SELECT 1+1; # This comment continues to the end of line""" + sql2 = """-- This comment continues to the end of line + SELECT 1+1; -- This comment continues to the end of line""" + sql3 = """/* this is an in-line comment */ + SELECT 1 /* this is an in-line comment */ + 1;/* this is an in-line comment */""" + self.assertEqual( + remove_comments(sql1, db_type="mysql"), + "SELECT 1+1; # This comment continues to the end of line", + ) + self.assertEqual( + remove_comments(sql2, db_type="mysql"), + "SELECT 1+1; -- This comment continues to the end of line", + ) + self.assertEqual(remove_comments(sql3, db_type="mysql"), "SELECT 1 + 1;") + + def test_extract_tables_by_sql_parse(self): + """ + 测试表解析 + :return: + """ + sql = "select * from user.users a join logs.log b on a.id=b.id;" + self.assertEqual( + extract_tables(sql), + [{"name": "users", "schema": "user"}, {"name": "log", "schema": "logs"}], + ) + + def test_generate_sql_from_sql(self): + """ + 测试从SQl文本中解析SQL + :return: + """ + text = "select * from sql_user;select * from sql_workflow;" + rows = generate_sql(text) + self.assertListEqual( + rows, + [ + {"sql_id": 1, "sql": "select * from sql_user;"}, + {"sql_id": 2, "sql": "select * from sql_workflow;"}, + ], + ) + + def test_generate_sql_from_xml(self): + """ + 测试从XML文本中解析SQL + :return: + """ + text = """ + + + + + """ + rows = generate_sql(text) + self.assertEqual( + rows, + [ + { + "sql_id": "testParameters", + "sql": "\nSELECT name,\n category,\n price\nFROM fruits\nWHERE category = ?\n AND price > ?", + } + ], + ) + + def test_get_full_sqlitem_list_anonymous_plsql(self): + """ + 测试SQL文本中plsql可执行块（匿名块）自动分割 + :return: + """ + text = """ +declare + v_rowcount integer; +begin + select count(1) into v_rowcount from user_tables + where table_name = upper('test2'); --账户特定关系人信息历史表 + if v_rowcount = 0 then + execute IMMEDIATE ' + create table test2 + ( + vc_bfcyid int, --受益人信息唯一标志ID + vc_specperid VARCHAR2(100) --特定关系人信息唯一标志ID + ) + '; + execute IMMEDIATE ' + CREATE index Idx_test2_1 ON test2(VC_BFCYID) + '; + end if; +end; +/ + +BEGIN + insert into test2 values(1,'qq1'); + commit; +END; +/ +""" + lists = get_full_sqlitem_list(text, "db") + rows = [ + SqlItem( + id=0, + statement="""declare + v_rowcount integer; +begin + select count(1) into v_rowcount from user_tables + where table_name = upper('test2'); --账户特定关系人信息历史表 + if v_rowcount = 0 then + execute IMMEDIATE ' + create table test2 + ( + vc_bfcyid int, --受益人信息唯一标志ID + vc_specperid VARCHAR2(100) --特定关系人信息唯一标志ID + ) + '; + execute IMMEDIATE ' + CREATE index Idx_test2_1 ON test2(VC_BFCYID) + '; + end if; +end;""", + stmt_type="PLSQL", + object_owner="db", + object_type="ANONYMOUS", + object_name="ANONYMOUS", + ), + SqlItem( + id=0, + statement="""BEGIN + insert into test2 values(1,'qq1'); + commit; +END;""", + stmt_type="PLSQL", + object_owner="db", + object_type="ANONYMOUS", + object_name="ANONYMOUS", + ), + ] + self.assertIsInstance(lists[0], SqlItem) + self.assertIsInstance(lists[1], SqlItem) + self.assertEqual(lists[0].__dict__, rows[0].__dict__) + self.assertEqual(lists[1].__dict__, rows[1].__dict__) + + def test_get_full_sqlitem_list_plsql(self): + """ + 测试SQL文本中plsql对象定义语句（存储过程、函数等）自动分割 + :return: + """ + text = """ +create or replace procedure INSERTUSER +(id IN NUMBER, +name IN VARCHAR2) +is +begin + insert into user1 values(id,name); +end; +/ + +create or replace function annual_income(name1 varchar2) +return number is +annual_salary number(7,2); +begin + select sal*12+nvl(comm,0) into annual_salary from emp where lower(ename)=lower(name1); +return annual_salary; +end; +/ +""" + lists = get_full_sqlitem_list(text, "db") + rows = [ + SqlItem( + id=0, + statement="""create or replace procedure INSERTUSER +(id IN NUMBER, +name IN VARCHAR2) +is +begin + insert into user1 values(id,name); +end;""", + stmt_type="PLSQL", + object_owner="db", + object_type="PROCEDURE", + object_name="INSERTUSER", + ), + SqlItem( + id=0, + statement="""create or replace function annual_income(name1 varchar2) +return number is +annual_salary number(7,2); +begin + select sal*12+nvl(comm,0) into annual_salary from emp where lower(ename)=lower(name1); +return annual_salary; +end;""", + stmt_type="PLSQL", + object_owner="db", + object_type="FUNCTION", + object_name="ANNUAL_INCOME", + ), + ] + self.assertIsInstance(lists[0], SqlItem) + self.assertIsInstance(lists[1], SqlItem) + self.assertEqual(lists[0].__dict__, rows[0].__dict__) + self.assertEqual(lists[1].__dict__, rows[1].__dict__) + + def test_get_full_sqlitem_list_sql_after_plsql(self): + """ + 测试SQL文本中plsql后面普通SQL语句以;自动分割 + :return: + """ + text = """ +create or replace procedure INSERTUSER +(id IN NUMBER, +name IN VARCHAR2) +is +begin + insert into user1 values(id,name); +end; +/ +update user_account set created=sysdate where account_no=1; +create table user( + id int, + uname varchar(100), + age int +); +""" + sql1 = "update user_account set created=sysdate where account_no=1;" + sql2 = """create table user( + id int, + uname varchar(100), + age int +);""" + lists = get_full_sqlitem_list(text, "db") + rows = [ + SqlItem( + id=0, + statement=sqlparse.format( + sql1, strip_comments=True, reindent=True, keyword_case="lower" + ), + stmt_type="SQL", + object_owner="", + object_type="", + object_name="", + ), + SqlItem( + id=0, + statement=sqlparse.format( + sql2, strip_comments=True, reindent=True, keyword_case="lower" + ), + stmt_type="SQL", + object_owner="", + object_type="", + object_name="", + ), + ] + self.assertIsInstance(lists[1], SqlItem) + self.assertIsInstance(lists[2], SqlItem) + self.assertEqual(lists[1].__dict__, rows[0].__dict__) + self.assertEqual(lists[2].__dict__, rows[1].__dict__) + + def test_get_full_sqlitem_list_sql(self): + """ + 测试普通SQL（不包含plsql执行块和plsql对象定义块）文本，以;符号进行SQL语句分割 + :return: + """ + text = """ +update user_account set created=sysdate where account_no=1; +create table user( + id int, + uname varchar(100), + age int +); +""" + sql1 = "update user_account set created=sysdate where account_no=1;" + sql2 = """create table user( + id int, + uname varchar(100), + age int +);""" + lists = get_full_sqlitem_list(text, "db") + rows = [ + SqlItem( + id=0, + statement=sqlparse.format( + sql1, strip_comments=True, reindent=True, keyword_case="lower" + ), + stmt_type="SQL", + object_owner="", + object_type="", + object_name="", + ), + SqlItem( + id=0, + statement=sqlparse.format( + sql2, strip_comments=True, reindent=True, keyword_case="lower" + ), + stmt_type="SQL", + object_owner="", + object_type="", + object_name="", + ), + ] + self.assertIsInstance(lists[0], SqlItem) + self.assertIsInstance(lists[1], SqlItem) + self.assertEqual(lists[0].__dict__, rows[0].__dict__) + self.assertEqual(lists[1].__dict__, rows[1].__dict__) + + def test_filter_with_string_list_match_regex(self): + """ + 测试：当 db_list 是字符串列表时，且正则表达式匹配的情况。 + """ + db_list = ["a_db", "b_db", "test_db", "prod_db"] + regex = r".*_db$" # 以 "_db" 结尾的数据库名称 + result = filter_db_list(db_list, regex, is_match_regex=True) + self.assertEqual(result, ["a_db", "b_db", "test_db", "prod_db"]) # 所有应该匹配 + + def test_filter_with_string_list_not_match_regex(self): + """ + 测试：当 db_list 是字符串列表时，且正则表达式不匹配的情况。 + """ + db_list = ["a_db", "b_db", "test_db", "prod_db", "invalid"] + regex = r".*_db$" + result = filter_db_list(db_list, regex, is_match_regex=False) + self.assertEqual(result, ["invalid"]) # 仅 "invalid" 不匹配正则 + + def test_filter_with_dict_list_match_regex(self): + """ + 测试：当 db_list 是字典列表时，且根据指定键进行匹配正则表达式。 + """ + db_list = [ + {"value": "0", "text": "0(11)"}, + {"value": "2", "text": "2(33)"}, + {"value": "4", "text": "4(3111)"}, + {"value": "11", "text": "11(3)"}, + {"value": "44", "text": "44(3)"}, + ] + regex = r"^(0|4|6|11|12|13)$" # 匹配 0, 4, 6, 11, 12, 13 的正则 + result = filter_db_list(db_list, regex, is_match_regex=True, key="value") + + # 期望返回匹配的字典项 + expected_result = [ + {"value": "0", "text": "0(11)"}, + {"value": "4", "text": "4(3111)"}, + {"value": "11", "text": "11(3)"}, + ] + + self.assertEqual(result, expected_result) # 验证返回结果是否符合预期 + + def test_filter_with_dict_list_not_match_regex(self): + """ + 测试：当 db_list 是字典列表时，且根据指定键不匹配正则表达式。 + """ + db_list = [ + {"value": "a_db"}, + {"value": "b_db"}, + {"value": "prod_db"}, + {"value": "invalid"}, + ] + regex = r".*_db$" + result = filter_db_list(db_list, regex, is_match_regex=False, key="value") + self.assertEqual(result, [{"value": "invalid"}]) + + def test_filter_without_regex(self): + """ + 测试：当没有提供正则表达式时，函数应返回原始的 db_list。 + """ + db_list = ["a_db", "b_db", "invalid"] + result = filter_db_list(db_list, "", is_match_regex=True) + self.assertEqual(result, db_list) # 没有正则，应该返回原始列表 + + def test_invalid_regex(self): + """ + 测试：提供无效正则表达式时，函数应抛出 ValueError 异常。 + """ + db_list = ["a_db", "b_db"] + regex = r"[unclosed_bracket" + with self.assertRaises(ValueError): + filter_db_list(db_list, regex, is_match_regex=True) + + def test_filter_with_match_and_not_match(self): + """ + 测试：使用不同的正则分别测试匹配和不匹配的情况。 + """ + db_list = ["test_db", "dmp_db", "za_db", "invalid_db", "prod_db", "no_match"] + + # 匹配正则：以 "test_db", "dmp_db", 或 "za" 开头的数据库名称 + match_regex = r"^(test_db|dmp_db|za.*)$" + + # 不匹配正则：以 "_db" 结尾的数据库名称 + not_match_regex = r".*_db$" + + # 测试匹配正则的情况 + match_result = filter_db_list(db_list, match_regex, is_match_regex=True) + self.assertEqual( + match_result, ["test_db", "dmp_db", "za_db"] + ) # 仅匹配 test_db, dmp_db, za_db + + # 测试不匹配正则的情况 + not_match_result = filter_db_list( + db_list, not_match_regex, is_match_regex=False + ) + self.assertEqual( + not_match_result, ["no_match"] + ) # 仅 no_match 不符合 "_db$" 规则 diff --git a/sql/utils/tests.py b/sql/utils/tests.py index 6ece20636b..512e0f5edd 100644 --- a/sql/utils/tests.py +++ b/sql/utils/tests.py @@ -44,347 +44,6 @@ __author__ = "hhyo" -class TestSQLUtils(TestCase): - def test_get_syntax_type(self): - """ - 测试语法判断 - :return: - """ - dml_sql = "select * from users;" - ddl_sql = "alter table users add id not null default 0 comment 'id' " - self.assertEqual(get_syntax_type(dml_sql), "DML") - self.assertEqual(get_syntax_type(ddl_sql), "DDL") - - def test_get_syntax_type_by_re(self): - """ - 测试语法判断，不使用sqlparse解析,直接正则匹配判断 - :return: - """ - dml_sql = "select * from users;" - ddl_sql = "alter table users add id int not null default 0 comment 'id' " - other_sql = "show engine innodb status" - self.assertEqual(get_syntax_type(dml_sql, parser=False, db_type="mysql"), "DML") - self.assertEqual(get_syntax_type(ddl_sql, parser=False, db_type="mysql"), "DDL") - self.assertIsNone(get_syntax_type(other_sql, parser=False, db_type="mysql")) - - def test_remove_comments(self): - """ - 测试去除SQL注释 - :return: - """ - sql1 = """ # This comment continues to the end of line - SELECT 1+1; # This comment continues to the end of line""" - sql2 = """-- This comment continues to the end of line - SELECT 1+1; -- This comment continues to the end of line""" - sql3 = """/* this is an in-line comment */ - SELECT 1 /* this is an in-line comment */ + 1;/* this is an in-line comment */""" - self.assertEqual( - remove_comments(sql1, db_type="mysql"), - "SELECT 1+1; # This comment continues to the end of line", - ) - self.assertEqual( - remove_comments(sql2, db_type="mysql"), - "SELECT 1+1; -- This comment continues to the end of line", - ) - self.assertEqual(remove_comments(sql3, db_type="mysql"), "SELECT 1 + 1;") - - def test_extract_tables_by_sql_parse(self): - """ - 测试表解析 - :return: - """ - sql = "select * from user.users a join logs.log b on a.id=b.id;" - self.assertEqual( - extract_tables(sql), - [{"name": "users", "schema": "user"}, {"name": "log", "schema": "logs"}], - ) - - def test_generate_sql_from_sql(self): - """ - 测试从SQl文本中解析SQL - :return: - """ - text = "select * from sql_user;select * from sql_workflow;" - rows = generate_sql(text) - self.assertListEqual( - rows, - [ - {"sql_id": 1, "sql": "select * from sql_user;"}, - {"sql_id": 2, "sql": "select * from sql_workflow;"}, - ], - ) - - def test_generate_sql_from_xml(self): - """ - 测试从XML文本中解析SQL - :return: - """ - text = """ - - - - - """ - rows = generate_sql(text) - self.assertEqual( - rows, - [ - { - "sql_id": "testParameters", - "sql": "\nSELECT name,\n category,\n price\nFROM fruits\nWHERE category = ?\n AND price > ?", - } - ], - ) - - def test_get_full_sqlitem_list_anonymous_plsql(self): - """ - 测试SQL文本中plsql可执行块（匿名块）自动分割 - :return: - """ - text = """ -declare - v_rowcount integer; -begin - select count(1) into v_rowcount from user_tables - where table_name = upper('test2'); --账户特定关系人信息历史表 - if v_rowcount = 0 then - execute IMMEDIATE ' - create table test2 - ( - vc_bfcyid int, --受益人信息唯一标志ID - vc_specperid VARCHAR2(100) --特定关系人信息唯一标志ID - ) - '; - execute IMMEDIATE ' - CREATE index Idx_test2_1 ON test2(VC_BFCYID) - '; - end if; -end; -/ - -BEGIN - insert into test2 values(1,'qq1'); - commit; -END; -/ -""" - lists = get_full_sqlitem_list(text, "db") - rows = [ - SqlItem( - id=0, - statement="""declare - v_rowcount integer; -begin - select count(1) into v_rowcount from user_tables - where table_name = upper('test2'); --账户特定关系人信息历史表 - if v_rowcount = 0 then - execute IMMEDIATE ' - create table test2 - ( - vc_bfcyid int, --受益人信息唯一标志ID - vc_specperid VARCHAR2(100) --特定关系人信息唯一标志ID - ) - '; - execute IMMEDIATE ' - CREATE index Idx_test2_1 ON test2(VC_BFCYID) - '; - end if; -end;""", - stmt_type="PLSQL", - object_owner="db", - object_type="ANONYMOUS", - object_name="ANONYMOUS", - ), - SqlItem( - id=0, - statement="""BEGIN - insert into test2 values(1,'qq1'); - commit; -END;""", - stmt_type="PLSQL", - object_owner="db", - object_type="ANONYMOUS", - object_name="ANONYMOUS", - ), - ] - self.assertIsInstance(lists[0], SqlItem) - self.assertIsInstance(lists[1], SqlItem) - self.assertEqual(lists[0].__dict__, rows[0].__dict__) - self.assertEqual(lists[1].__dict__, rows[1].__dict__) - - def test_get_full_sqlitem_list_plsql(self): - """ - 测试SQL文本中plsql对象定义语句（存储过程、函数等）自动分割 - :return: - """ - text = """ -create or replace procedure INSERTUSER -(id IN NUMBER, -name IN VARCHAR2) -is -begin - insert into user1 values(id,name); -end; -/ - -create or replace function annual_income(name1 varchar2) -return number is -annual_salary number(7,2); -begin - select sal*12+nvl(comm,0) into annual_salary from emp where lower(ename)=lower(name1); -return annual_salary; -end; -/ -""" - lists = get_full_sqlitem_list(text, "db") - rows = [ - SqlItem( - id=0, - statement="""create or replace procedure INSERTUSER -(id IN NUMBER, -name IN VARCHAR2) -is -begin - insert into user1 values(id,name); -end;""", - stmt_type="PLSQL", - object_owner="db", - object_type="PROCEDURE", - object_name="INSERTUSER", - ), - SqlItem( - id=0, - statement="""create or replace function annual_income(name1 varchar2) -return number is -annual_salary number(7,2); -begin - select sal*12+nvl(comm,0) into annual_salary from emp where lower(ename)=lower(name1); -return annual_salary; -end;""", - stmt_type="PLSQL", - object_owner="db", - object_type="FUNCTION", - object_name="ANNUAL_INCOME", - ), - ] - self.assertIsInstance(lists[0], SqlItem) - self.assertIsInstance(lists[1], SqlItem) - self.assertEqual(lists[0].__dict__, rows[0].__dict__) - self.assertEqual(lists[1].__dict__, rows[1].__dict__) - - def test_get_full_sqlitem_list_sql_after_plsql(self): - """ - 测试SQL文本中plsql后面普通SQL语句以;自动分割 - :return: - """ - text = """ -create or replace procedure INSERTUSER -(id IN NUMBER, -name IN VARCHAR2) -is -begin - insert into user1 values(id,name); -end; -/ -update user_account set created=sysdate where account_no=1; -create table user( - id int, - uname varchar(100), - age int -); -""" - sql1 = "update user_account set created=sysdate where account_no=1;" - sql2 = """create table user( - id int, - uname varchar(100), - age int -);""" - lists = get_full_sqlitem_list(text, "db") - rows = [ - SqlItem( - id=0, - statement=sqlparse.format( - sql1, strip_comments=True, reindent=True, keyword_case="lower" - ), - stmt_type="SQL", - object_owner="", - object_type="", - object_name="", - ), - SqlItem( - id=0, - statement=sqlparse.format( - sql2, strip_comments=True, reindent=True, keyword_case="lower" - ), - stmt_type="SQL", - object_owner="", - object_type="", - object_name="", - ), - ] - self.assertIsInstance(lists[1], SqlItem) - self.assertIsInstance(lists[2], SqlItem) - self.assertEqual(lists[1].__dict__, rows[0].__dict__) - self.assertEqual(lists[2].__dict__, rows[1].__dict__) - - def test_get_full_sqlitem_list_sql(self): - """ - 测试普通SQL（不包含plsql执行块和plsql对象定义块）文本，以;符号进行SQL语句分割 - :return: - """ - text = """ -update user_account set created=sysdate where account_no=1; -create table user( - id int, - uname varchar(100), - age int -); -""" - sql1 = "update user_account set created=sysdate where account_no=1;" - sql2 = """create table user( - id int, - uname varchar(100), - age int -);""" - lists = get_full_sqlitem_list(text, "db") - rows = [ - SqlItem( - id=0, - statement=sqlparse.format( - sql1, strip_comments=True, reindent=True, keyword_case="lower" - ), - stmt_type="SQL", - object_owner="", - object_type="", - object_name="", - ), - SqlItem( - id=0, - statement=sqlparse.format( - sql2, strip_comments=True, reindent=True, keyword_case="lower" - ), - stmt_type="SQL", - object_owner="", - object_type="", - object_name="", - ), - ] - self.assertIsInstance(lists[0], SqlItem) - self.assertIsInstance(lists[1], SqlItem) - self.assertEqual(lists[0].__dict__, rows[0].__dict__) - self.assertEqual(lists[1].__dict__, rows[1].__dict__) - - class TestSQLReview(TestCase): """ 测试sql review内的方法 diff --git a/sql_api/api_instance.py b/sql_api/api_instance.py index 1e3b3931a1..6e21e0e879 100644 --- a/sql_api/api_instance.py +++ b/sql_api/api_instance.py @@ -3,6 +3,7 @@ from drf_spectacular.utils import extend_schema from sql.utils.sql_utils import filter_db_list + from .serializers import ( InstanceSerializer, InstanceDetailSerializer, From 7ca276b889df6b5631dfd0b561eb7e0d96c1e399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 17:04:36 +0800 Subject: [PATCH 24/28] sql_utils --- sql/utils/sql_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/utils/sql_utils.py b/sql/utils/sql_utils.py index 8cf30df0f3..36eed15f36 100644 --- a/sql/utils/sql_utils.py +++ b/sql/utils/sql_utils.py @@ -414,5 +414,3 @@ def filter_db_list(db_list, db_name_regex: str, is_match_regex: bool, key="value if (is_match_regex and is_match) or (not is_match_regex and not is_match): filtered_list.append(db) return filtered_list - - From 1b7cd7373b08437edd1990201dc6aef4af1d31e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 17:07:28 +0800 Subject: [PATCH 25/28] resource.rows --- sql/instance.py | 8 ++++---- sql_api/api_instance.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/instance.py b/sql/instance.py index a9b9332e2e..852c16789e 100644 --- a/sql/instance.py +++ b/sql/instance.py @@ -333,11 +333,11 @@ def instance_resource(request): tb_name = query_engine.escape_string(tb_name) if resource_type == "database": resource = query_engine.get_all_databases() - resource = filter_db_list( - resource, query_engine.instance.show_db_name_regex, True + resource.rows = filter_db_list( + resource.rows, query_engine.instance.show_db_name_regex, True ) - resource = filter_db_list( - resource, query_engine.instance.denied_db_name_regex, False + resource.rows = filter_db_list( + resource.rows, query_engine.instance.denied_db_name_regex, False ) elif resource_type == "schema" and db_name: resource = query_engine.get_all_schemas(db_name=db_name) diff --git a/sql_api/api_instance.py b/sql_api/api_instance.py index 6e21e0e879..f771ad204a 100644 --- a/sql_api/api_instance.py +++ b/sql_api/api_instance.py @@ -196,11 +196,11 @@ def post(self, request): tb_name = query_engine.escape_string(tb_name) if resource_type == "database": resource = query_engine.get_all_databases() - resource = filter_db_list( - resource, query_engine.instance.show_db_name_regex, True + resource.rows = filter_db_list( + resource.rows, query_engine.instance.show_db_name_regex, True ) - resource = filter_db_list( - resource, query_engine.instance.denied_db_name_regex, False + resource.rows = filter_db_list( + resource.rows, query_engine.instance.denied_db_name_regex, False ) elif resource_type == "schema" and db_name: resource = query_engine.get_all_schemas(db_name=db_name) From 978b85ab8d6d12f3acfdbcf93d696d3369bf512c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 17:12:44 +0800 Subject: [PATCH 26/28] =?UTF-8?q?=E5=9B=9E=E6=BB=9Amysql.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/engines/mysql.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sql/engines/mysql.py b/sql/engines/mysql.py index c34b865bc8..369931791b 100644 --- a/sql/engines/mysql.py +++ b/sql/engines/mysql.py @@ -12,10 +12,7 @@ from schemaobject.connection import build_database_url from sql.engines.goinception import GoInceptionEngine -from sql.utils.sql_utils import ( - get_syntax_type, - remove_comments, -) +from sql.utils.sql_utils import get_syntax_type, remove_comments from . import EngineBase from .models import ResultSet, ReviewResult, ReviewSet from sql.utils.data_masking import data_masking From a03674ee198d0813480b2bfec11b414113b8551e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 17:14:17 +0800 Subject: [PATCH 27/28] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql_api/api_instance.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql_api/api_instance.py b/sql_api/api_instance.py index f771ad204a..c0fc9c45ba 100644 --- a/sql_api/api_instance.py +++ b/sql_api/api_instance.py @@ -1,9 +1,7 @@ from rest_framework import views, generics, status, serializers from rest_framework.response import Response from drf_spectacular.utils import extend_schema - from sql.utils.sql_utils import filter_db_list - from .serializers import ( InstanceSerializer, InstanceDetailSerializer, From 5e696823b15f69aca7482a2b11a0d70e060e71a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=A3=9E?= Date: Thu, 19 Sep 2024 17:39:07 +0800 Subject: [PATCH 28/28] =?UTF-8?q?=E6=94=B9=E4=B8=BA=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sql/instance.py | 8 ++++++-- sql_api/api_instance.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sql/instance.py b/sql/instance.py index 852c16789e..0b0afffecd 100644 --- a/sql/instance.py +++ b/sql/instance.py @@ -334,10 +334,14 @@ def instance_resource(request): if resource_type == "database": resource = query_engine.get_all_databases() resource.rows = filter_db_list( - resource.rows, query_engine.instance.show_db_name_regex, True + db_list=resource.rows, + db_name_regex=query_engine.instance.show_db_name_regex, + is_match_regex=True, ) resource.rows = filter_db_list( - resource.rows, query_engine.instance.denied_db_name_regex, False + db_list=resource.rows, + db_name_regex=query_engine.instance.denied_db_name_regex, + is_match_regex=False, ) elif resource_type == "schema" and db_name: resource = query_engine.get_all_schemas(db_name=db_name) diff --git a/sql_api/api_instance.py b/sql_api/api_instance.py index c0fc9c45ba..6624a9467a 100644 --- a/sql_api/api_instance.py +++ b/sql_api/api_instance.py @@ -195,10 +195,14 @@ def post(self, request): if resource_type == "database": resource = query_engine.get_all_databases() resource.rows = filter_db_list( - resource.rows, query_engine.instance.show_db_name_regex, True + db_list=resource.rows, + db_name_regex=query_engine.instance.show_db_name_regex, + is_match_regex=True, ) resource.rows = filter_db_list( - resource.rows, query_engine.instance.denied_db_name_regex, False + db_list=resource.rows, + db_name_regex=query_engine.instance.denied_db_name_regex, + is_match_regex=False, ) elif resource_type == "schema" and db_name: resource = query_engine.get_all_schemas(db_name=db_name)