Skip to content

Commit

Permalink
add cassandra draft
Browse files Browse the repository at this point in the history 
add cassandra integration test

重构 engine 的代码, 改为动态加载
  • Loading branch information
@LeoQuote
LeoQuote committed Aug 14, 2023
1 parent 4afe3e6 commit 8dcd09f
Show file tree
Hide file tree
Showing 27 changed files with 562 additions and 253 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/django.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
strategy:
max-parallel: 4
matrix:
python-version: [3.8, 3.9, "3.10"]
python-version: ["3.8", "3.9", "3.10", "3.11"]

# https://github.com/actions/example-services/tree/master/.github/workflows
services:
Expand Down Expand Up @@ -70,7 +70,7 @@ jobs:

- name: Install Dependencies
run: |
sudo apt-get update && sudo apt-get install libsasl2-dev libldap2-dev libssl-dev unixodbc unixodbc-dev
sudo apt-get update && sudo apt-get install libsasl2-dev libkrb5-dev libldap2-dev libssl-dev unixodbc unixodbc-dev
python -m pip install --upgrade pip
pip install codecov coverage flake8 -r requirements.txt
Expand Down
33 changes: 31 additions & 2 deletions archery/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
environ.Env.read_env(os.path.join(BASE_DIR, ".env"))
env = environ.Env(
DEBUG=(bool, False),
ALLOWED_HOSTS=(List[str], ["*"]),
ALLOWED_HOSTS=(list, ["*"]),
SECRET_KEY=(str, "hfusaf2m4ot#7)fkw#di2bu6(cv0@opwmafx5n#6=3d%x^hpl6"),
DATABASE_URL=(str, "mysql://root:@127.0.0.1:3306/archery"),
CACHE_URL=(str, "redis://127.0.0.1:6379/0"),
Expand All @@ -38,6 +38,21 @@
Q_CLUISTER_SYNC=(bool, False), # qcluster 同步模式, debug 时可以调整为 True
# CSRF_TRUSTED_ORIGINS=subdomain.example.com,subdomain.example2.com subdomain.example.com
CSRF_TRUSTED_ORIGINS=(list, []),
ENABLED_ENGINES=(
list,
[
"mysql",
"clickhouse",
"goinception",
"mssql",
"redis",
"pqsql",
"oracle",
"mongo",
"phoenix",
"odps",
],
),
)

# SECURITY WARNING: keep the secret key used in production secret!
Expand All @@ -57,6 +72,21 @@
# 请求限制
DATA_UPLOAD_MAX_MEMORY_SIZE = 15728640

AVAILABLE_ENGINES = {
"mysql": {"path": "sql.engines.mysql:MysqlEngine"},
"cassandra": {"path": "sql.engines.cassandra:CassandraEngine"},
"clickhouse": {"path": "sql.engines.clickhouse:ClickHouseEngine"},
"goinception": {"path": "sql.engines.goinception:GoInceptionEngine"},
"mssql": {"path": "sql.engines.mssql:MssqlEngine"},
"redis": {"path": "sql.engines.redis:RedisEngine"},
"pqsql": {"path": "sql.engines.pgsql:PgSQLEngine"},
"oracle": {"path": "sql.engines.oracle:OracleEngine"},
"mongo": {"path": "sql.engines.mongo:MongoEngine"},
"phoenix": {"path": "sql.engines.phoenix:PhoenixEngine"},
"odps": {"path": "sql.engines.odps:ODPSEngine"},
}
ENABLED_ENGINES = env("ENABLED_ENGINES")

# Application definition
INSTALLED_APPS = (
"django.contrib.admin",
Expand Down Expand Up @@ -245,7 +275,6 @@
ENABLE_OIDC = env("ENABLE_OIDC", False)
if ENABLE_OIDC:
INSTALLED_APPS += ("mozilla_django_oidc",)
MIDDLEWARE += ("mozilla_django_oidc.middleware.SessionRefresh",)
AUTHENTICATION_BACKENDS = (
"common.authenticate.oidc_auth.OIDCAuthenticationBackend",
"django.contrib.auth.backends.ModelBackend",
Expand Down
8 changes: 4 additions & 4 deletions downloads/dictionary/test_instance_test_archery.html
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<html>
<meta charset="utf-8">
<title>数据库表结构说明文档</title>
<title>数据库表结构说明文档</title>
<style>
body,td,th {font-family:"宋体"; font-size:12px;}
body,td,th {font-family:"宋体"; font-size:12px;}
table,h1,p{width:960px;margin:0px auto;}
table{border-collapse:collapse;border:1px solid #CCC;background:#efefef;}
table caption{text-align:left; background-color:#fff; line-height:2em; font-size:14px; font-weight:bold; }
Expand All @@ -11,8 +11,8 @@
</style>

<body>
<h1 style="text-align:center;">test_archery 数据字典 (共 0 个表)</h1>
<p style="text-align:center;margin:20px auto;">生成时间：2023-01-31 14:41:33</p>
<h1 style="text-align:center;">test_archery 数据字典 (共 0 个表)</h1>
<p style="text-align:center;margin:20px auto;">生成时间：2023-08-14 18:44:43</p>

</body>
</html>
5 changes: 3 additions & 2 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pyecharts==1.9.1
aliyun-python-sdk-rds==2.1.1
cx-Oracle==7.3.0
supervisor==4.1.0
phoenixdb==0.7
phoenixdb==1.2.1
django-mirage-field==1.4.0
schema-sync==0.9.7
parsedatetime==2.4
Expand All @@ -38,4 +38,5 @@ django-environ==0.8.1
alibabacloud_dysmsapi20170525==2.0.9
tencentcloud-sdk-python==3.0.656
mozilla-django-oidc==3.0.0
django-auth-dingding==0.0.2
django-auth-dingding==0.0.2
cassandra-driver
24 changes: 20 additions & 4 deletions sql/data_dictionary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,16 @@ def table_info(request):
)


def get_export_full_path(base_dir: str, instance_name: str, db_name: str) -> str:
"""validate if the instance_name and db_name provided is secure"""
fullpath = os.path.normpath(
os.path.join(base_dir, f"{instance_name}_{db_name}.html")
)
if not fullpath.startswith(base_dir):
return ""
return fullpath


@permission_required("sql.data_dictionary_export", raise_exception=True)
def export(request):
"""导出数据字典"""
Expand All @@ -111,7 +121,7 @@ def export(request):
elif request.user.is_superuser:
dbs = query_engine.get_all_databases().rows
else:
return JsonResponse({"status": 1, "msg": f"仅管理员可以导出整个实例的字典信息！", "data": []})
return JsonResponse({"status": 1, "msg": "仅管理员可以导出整个实例的字典信息！", "data": []})

# 获取数据，存入目录
path = os.path.join(settings.BASE_DIR, "downloads/dictionary")
Expand All @@ -126,12 +136,18 @@ def export(request):
data = loader.render_to_string(
template_name="dictionaryexport.html", context=context, request=request
)
with open(f"{path}/{instance_name}_{db}.html", "w") as f:
f.write(data)
fullpath = get_export_full_path(path, instance_name, db)
if not fullpath:
return JsonResponse({"status": 1, "msg": "实例名或db名不合法", "data": []})
with open(fullpath, "w", encoding="utf-8") as fp:
fp.write(data)
# 关闭连接
query_engine.close()
if db_name:
response = FileResponse(open(f"{path}/{instance_name}_{db_name}.html", "rb"))
fullpath = get_export_full_path(path, instance_name, db)
if not fullpath:
return JsonResponse({"status": 1, "msg": "实例名或db名不合法", "data": []})
response = FileResponse(open(fullpath, "rb"))
response["Content-Type"] = "application/octet-stream"
response[
"Content-Disposition"
Expand Down
80 changes: 28 additions & 52 deletions sql/engines/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
"""engine base库, 包含一个``EngineBase`` class和一个get_engine函数"""
import importlib
from sql.engines.models import ResultSet, ReviewSet
from sql.utils.ssh_tunnel import SSHConnection
from django.conf import settings


class EngineBase:
"""enginebase 只定义了init函数和若干方法的名字, 具体实现用mysql.py pg.py等实现"""

test_query = None

name = "Base"
info = "base engine"

def __init__(self, instance=None):
self.conn = None
self.thread_id = None
Expand Down Expand Up @@ -77,16 +82,6 @@ def test_connection(self):
"""测试实例链接是否正常"""
return self.query(sql=self.test_query)

@property
def name(self):
"""返回engine名称"""
return "base"

@property
def info(self):
"""返回引擎简介"""
return "Base engine"

def escape_string(self, value: str) -> str:
"""参数转义"""
return value
Expand Down Expand Up @@ -179,7 +174,7 @@ def query(
limit_num=0,
close_conn=True,
parameters=None,
**kwargs
**kwargs,
):
"""实际查询 返回一个ResultSet"""
return ResultSet()
Expand Down Expand Up @@ -213,6 +208,22 @@ def set_variable(self, variable_name, variable_value):
return ResultSet()


def get_engine_map():
available_engines = settings.AVAILABLE_ENGINES
enabled_engines = {}
for e in settings.ENABLED_ENGINES:
config = available_engines.get(e)
if not config:
raise ValueError(f"invalid engine {e}, not found in engine map")
module, o = config["path"].split(":")
engine = getattr(importlib.import_module(module), o)
enabled_engines[e] = engine
return enabled_engines


engine_map = get_engine_map()


def get_engine(instance=None): # pragma: no cover
"""获取数据库操作engine"""
if instance.db_type == "mysql":
Expand All @@ -222,44 +233,9 @@ def get_engine(instance=None): # pragma: no cover
from .cloud.aliyun_rds import AliyunRDS

return AliyunRDS(instance=instance)
from .mysql import MysqlEngine

return MysqlEngine(instance=instance)
elif instance.db_type == "mssql":
from .mssql import MssqlEngine

return MssqlEngine(instance=instance)
elif instance.db_type == "redis":
from .redis import RedisEngine

return RedisEngine(instance=instance)
elif instance.db_type == "pgsql":
from .pgsql import PgSQLEngine

return PgSQLEngine(instance=instance)
elif instance.db_type == "oracle":
from .oracle import OracleEngine

return OracleEngine(instance=instance)
elif instance.db_type == "mongo":
from .mongo import MongoEngine

return MongoEngine(instance=instance)
elif instance.db_type == "goinception":
from .goinception import GoInceptionEngine

return GoInceptionEngine(instance=instance)
elif instance.db_type == "phoenix":
from .phoenix import PhoenixEngine

return PhoenixEngine(instance=instance)

elif instance.db_type == "odps":
from .odps import ODPSEngine

return ODPSEngine(instance=instance)

elif instance.db_type == "clickhouse":
from .clickhouse import ClickHouseEngine

return ClickHouseEngine(instance=instance)
engine = engine_map.get(instance.db_type)
if not engine:
raise ValueError(
f"engine {instance.db_type} not enabled or not supported, please contact admin"
)
return engine(instance=instance)
Loading

0 comments on commit 8dcd09f

Please sign in to comment.