From 37aedfc87c4e39015bd456bf34debe5a3a4cfeb3 Mon Sep 17 00:00:00 2001 From: MugdhaHardikar-GSLab Date: Tue, 29 Mar 2022 15:22:57 +0530 Subject: [PATCH 1/6] feat(s3): add s3 source (#4490) * feat(data-lake): add containers and folder level dataset support * docs(data-lake): Update readme for data lake * doc(data-lake): fix examples, update doc * lint fix * feat(s3): add s3 source, restore old data-lake source Co-authored-by: Mayuri N --- .../examples/recipes/s3_to_file.yml | 19 + metadata-ingestion/setup.py | 13 +- .../source_docs/s3_data_lake.md | 180 + .../src/datahub/emitter/mcp_builder.py | 10 +- .../datahub/ingestion/source/aws/s3_util.py | 12 + .../datahub/ingestion/source/s3/__init__.py | 717 ++++ .../src/datahub/ingestion/source/s3/config.py | 174 + .../datahub/ingestion/source/s3/profiling.py | 532 +++ .../src/datahub/ingestion/source/s3/report.py | 17 + .../integration/s3/data_lake_mces_golden.json | 3030 +++++++++++++++++ .../golden_mces_folder_no_partition.json | 957 ++++++ ...lden_mces_folder_no_partition_exclude.json | 774 +++++ ...den_mces_folder_no_partition_filename.json | 774 +++++ .../golden_mces_folder_no_partition_glob.json | 774 +++++ .../golden_mces_folder_partition_basic.json | 1381 ++++++++ .../golden_mces_folder_partition_keyval.json | 1381 ++++++++ .../local/golden_mces_multiple_files.json | 2724 +++++++++++++++ .../local/golden_mces_single_file.json | 774 +++++ .../s3/golden_mces_folder_no_partition.json | 615 ++++ ...lden_mces_folder_no_partition_exclude.json | 451 +++ ...den_mces_folder_no_partition_filename.json | 451 +++ .../golden_mces_folder_no_partition_glob.json | 451 +++ .../golden_mces_folder_partition_basic.json | 1058 ++++++ .../golden_mces_folder_partition_keyval.json | 1058 ++++++ .../s3/golden_mces_multiple_files.json | 2287 +++++++++++++ .../s3/golden_mces_single_file.json | 451 +++ .../s3/sources/s3/folder_no_partition.json | 17 + .../s3/folder_no_partition_exclude.json | 20 + .../s3/folder_no_partition_filename.json | 19 + .../sources/s3/folder_no_partition_glob.json | 20 + .../s3/sources/s3/folder_partition_basic.json | 21 + .../sources/s3/folder_partition_keyval.json | 18 + .../s3/sources/s3/multiple_files.json | 19 + .../s3/sources/s3/single_file.json | 16 + .../cases_1/chord_progressions_avro.avro | Bin 0 -> 1024 bytes .../cases_1/chord_progressions_csv.csv | 29 + .../s3/test_data/cases_1/countries_json.json | 1 + .../s3/test_data/cases_1/food_avro.avro | Bin 0 -> 497 bytes .../s3/test_data/cases_1/food_csv.csv | 71 + .../s3/test_data/cases_1/food_parquet.parquet | Bin 0 -> 4206 bytes ...PS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv | 34 + .../cases_2/pokemon_abilities_json.json | 1 + .../s3/test_data/cases_2/small.csv | 6 + .../wa_fn_usec_hr_employee_attrition_csv.csv | 250 ++ ...PS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv | 34 + .../folder_aaa/chord_progressions_avro.avro | Bin 0 -> 1024 bytes .../folder_aaa/chord_progressions_csv.csv | 29 + .../folder_aa/folder_aaa/countries_json.json | 1 + .../month=feb/part1.json | 1 + .../month=feb/part2.json | 1 + .../month=jan/part1.json | 1 + .../month=jan/part2.json | 1 + .../month=feb/part1.json | 1 + .../month=feb/part2.json | 1 + .../month=march/part1.json | 1 + .../month=march/part2.json | 1 + .../month=april/part1.json | 1 + .../month=april/part2.json | 1 + .../month=march/part1.json | 1 + .../month=march/part2.json | 1 + .../folder_aa/folder_aaa/food_csv/part1.csv | 71 + .../folder_aa/folder_aaa/food_csv/part2.csv | 71 + .../folder_aa/folder_aaa/food_parquet.parquet | Bin 0 -> 4206 bytes .../folder_aaa/food_parquet/part1.parquet | Bin 0 -> 4206 bytes .../folder_aaa/food_parquet/part2.parquet | Bin 0 -> 4206 bytes .../year=2019/month=feb/part1.json | 1 + .../year=2019/month=feb/part2.json | 1 + .../year=2019/month=jan/part1.json | 1 + .../year=2019/month=jan/part2.json | 1 + .../year=2020/month=feb/part1.json | 1 + .../year=2020/month=feb/part2.json | 1 + .../year=2020/month=march/part1.json | 1 + .../year=2020/month=march/part2.json | 1 + .../year=2021/month=april/part1.json | 1 + .../year=2021/month=april/part2.json | 1 + .../year=2021/month=march/part1.json | 1 + .../year=2021/month=march/part2.json | 1 + .../folder_a/folder_aa/folder_aaa/small.csv | 6 + .../wa_fn_usec_hr_employee_attrition_csv.csv | 250 ++ .../tests/integration/s3/test_s3.py | 194 ++ 80 files changed, 22284 insertions(+), 3 deletions(-) create mode 100644 metadata-ingestion/examples/recipes/s3_to_file.yml create mode 100644 metadata-ingestion/source_docs/s3_data_lake.md create mode 100644 metadata-ingestion/src/datahub/ingestion/source/s3/__init__.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/s3/config.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/s3/report.py create mode 100644 metadata-ingestion/tests/integration/s3/data_lake_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json create mode 100644 metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_exclude.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_filename.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_glob.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_basic.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_keyval.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/multiple_files.json create mode 100644 metadata-ingestion/tests/integration/s3/sources/s3/single_file.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_1/chord_progressions_avro.avro create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_1/chord_progressions_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_1/countries_json.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_1/food_avro.avro create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_1/food_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_1/food_parquet.parquet create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_2/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_2/pokemon_abilities_json.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_2/small.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/cases_2/wa_fn_usec_hr_employee_attrition_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/countries_json.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=feb/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=feb/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=jan/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2019/month=jan/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=feb/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=feb/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=march/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2020/month=march/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=april/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=april/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=march/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/folder_aaaa/pokemon_abilities_yearwise_2021/month=march/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part1.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv/part2.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet.parquet create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet/part1.parquet create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet/part2.parquet create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=feb/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=feb/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=jan/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2019/month=jan/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=feb/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=feb/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=march/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2020/month=march/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=april/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=april/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=march/part1.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json/year=2021/month=march/part2.json create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/small.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv create mode 100644 metadata-ingestion/tests/integration/s3/test_s3.py diff --git a/metadata-ingestion/examples/recipes/s3_to_file.yml b/metadata-ingestion/examples/recipes/s3_to_file.yml new file mode 100644 index 00000000000000..348fe4f1d0d031 --- /dev/null +++ b/metadata-ingestion/examples/recipes/s3_to_file.yml @@ -0,0 +1,19 @@ +--- +# see https://datahubproject.io/docs/metadata-ingestion/source_docs/s3_data_lake for complete documentation +source: + type: "s3" + config: + platform: s3 + path_spec: + include: "s3://covid19-lake/covid_knowledge_graph/csv/nodes/*.*" + aws_config: + aws_access_key_id: accessKey + aws_secret_access_key: secretKey + aws_region: us-east-2 + +# see https://datahubproject.io/docs/metadata-ingestion/sink_docs/file for complete documentation +sink: + type: "file" + config: + filename: "./s3_data_lake_mces.json" + diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 05d96a4b3f2e11..5c7d78d0dcbae7 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -113,7 +113,7 @@ def get_long_description(): "tableschema>=1.20.2", "ujson>=4.3.0", "types-ujson>=4.2.1", - "smart-open[s3]>=5.2.1", + "smart-open[s3]>=5.2.1" } data_lake_profiling = { @@ -121,6 +121,12 @@ def get_long_description(): "pyspark==3.0.3", } +s3_base = { + *data_lake_base, + "moto[s3]", + "wcmatch", +} + # Note: for all of these, framework_common will be added. plugins: Dict[str, Set[str]] = { # Sink plugins. @@ -142,6 +148,7 @@ def get_long_description(): "datahub-lineage-file": set(), "datahub-business-glossary": set(), "data-lake": {*data_lake_base, *data_lake_profiling}, + "s3": {*s3_base, *data_lake_profiling}, "dbt": {"requests"}, "druid": sql_common | {"pydruid>=0.6.2"}, # Starting with 7.14.0 python client is checking if it is connected to elasticsearch client. If its not it throws @@ -224,7 +231,7 @@ def get_long_description(): *base_requirements, *framework_common, *mypy_stubs, - *data_lake_base, + *s3_base, "black>=21.12b0", "coverage>=5.1", "flake8>=3.8.3", @@ -267,6 +274,7 @@ def get_long_description(): "redshift", "redshift-usage", "data-lake", + "s3", "tableau", "trino", "hive", @@ -335,6 +343,7 @@ def get_long_description(): "clickhouse = datahub.ingestion.source.sql.clickhouse:ClickHouseSource", "clickhouse-usage = datahub.ingestion.source.usage.clickhouse_usage:ClickHouseUsageSource", "data-lake = datahub.ingestion.source.data_lake:DataLakeSource", + "s3 = datahub.ingestion.source.s3:S3Source", "dbt = datahub.ingestion.source.dbt:DBTSource", "druid = datahub.ingestion.source.sql.druid:DruidSource", "elasticsearch = datahub.ingestion.source.elastic_search:ElasticsearchSource", diff --git a/metadata-ingestion/source_docs/s3_data_lake.md b/metadata-ingestion/source_docs/s3_data_lake.md new file mode 100644 index 00000000000000..9a4114286ace9f --- /dev/null +++ b/metadata-ingestion/source_docs/s3_data_lake.md @@ -0,0 +1,180 @@ +# S3 Data Lake + +For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md). + +:::caution + +This source is in **Beta** and under active development. Not yet considered ready for production. + +::: + +## Setup + +To install this plugin, run `pip install 'acryl-datahub[s3]'`. Note that because the profiling is run with PySpark, we require Spark 3.0.3 with Hadoop 3.2 to be installed (see [compatibility](#compatibility) for more details). If profiling, make sure that permissions for **s3a://** access are set because Spark and Hadoop use the s3a:// protocol to interface with AWS (schema inference outside of profiling requires s3:// access). + +The s3 connector extracts schemas and profiles from a variety of file formats (see below for an exhaustive list). +Based on configuration, individual files or folders are ingested as tables, and profiles are computed similar to the [SQL profiler](./sql_profiles.md). +Enabling profiling will slow down ingestion runs. + +## Capabilities + +Extracts: + +- Row and column counts for each table +- For each column, if profiling is enabled: + - null counts and proportions + - distinct counts and proportions + - minimum, maximum, mean, median, standard deviation, some quantile values + - histograms or frequencies of unique values + +This connector supports both local files as well as those stored on AWS S3 (which must be identified using the prefix `s3://`). Supported file types are as follows: + +- CSV +- TSV +- JSON +- Parquet +- Apache Avro + +Schemas for Parquet and Avro files are extracted as provided. + +Schemas for schemaless formats (CSV, TSV, JSON) are inferred. For CSV and TSV files, we consider the first 100 rows by default, which can be controlled via the `max_rows` recipe parameter (see [below](#config-details)) +JSON file schemas are inferred on the basis of the entire file (given the difficulty in extracting only the first few objects of the file), which may impact performance. +We are working on using iterator-based JSON parsers to avoid reading in the entire JSON object. + + +| Capability | Status | Details | +|-------------------|--------|------------------------------------------| +| Platform Instance | ✔️ | [link](../../docs/platform-instances.md) | + + +## Quickstart recipe + +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + +For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes). + +```yml +source: + type: s3 + config: + path_spec: + include: "s3://covid19-lake/covid_knowledge_graph/csv/nodes/*.*" + aws_config: + aws_access_key_id: ***** + aws_secret_access_key: ***** + aws_region: us-east-2 + env: "PROD" + profiling: + enabled: false + +sink: + # sink configs +``` + +## Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +|------------------------------------------------------|--------------------------|-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `path_spec.include` | ✅ | | Path to table (s3 or local file system). Name variable {table} is used to mark the folder with dataset. In absence of {table}, file level dataset will be created. Check below examples for more details. | +| `path_spec.exclude` | | | list of paths in glob pattern which will be excluded while scanning for the datasets | +| `path_spec.table_name` | | {table} | Display name of the dataset.Combination of named variableds from include path and strings | +| `path_spec.file_types` | | ["csv", "tsv", "json", "parquet", "avro"] | Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted. | +| `env` | | `PROD` | Environment to use in namespace when constructing URNs. | +| `platform` | | Autodetected | Platform to use in namespace when constructing URNs. If left blank, local paths will correspond to `file` and S3 paths will correspond to `s3`. | +| `platform_instance` | | | Platform instance for datasets and containers | +| `spark_driver_memory` | | `4g` | Max amount of memory to grant Spark. | +| `aws_config.aws_region` | If ingesting from AWS S3 | | AWS region code. | +| `aws_config.aws_access_key_id` | | Autodetected (Required for s3 profiling) | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `aws_config.aws_secret_access_key` | | Autodetected (Required for s3 profiling) | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `aws_config.aws_session_token` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `max_rows` | | `100` | Maximum number of rows to use when inferring schemas for TSV and CSV files. | +| `profile_patterns.allow` | | `*` | List of regex patterns for tables to profile (a must also be ingested for profiling). Defaults to all. | +| `profile_patterns.deny` | | | List of regex patterns for tables to not profile (a must also be ingested for profiling). Defaults to none. | +| `profile_patterns.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching of tables to profile. | +| `profiling.enabled` | | `False` | Whether profiling should be done. | +| `profiling.profile_table_level_only` | | `False` | Whether to perform profiling at table-level only or include column-level profiling as well. | +| `profiling.max_number_of_fields_to_profile` | | `None` | A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up. | +| `profiling.include_field_null_count` | | `True` | Whether to profile for the number of nulls for each column. | +| `profiling.include_field_min_value` | | `True` | Whether to profile for the min value of numeric columns. | +| `profiling.include_field_max_value` | | `True` | Whether to profile for the max value of numeric columns. | +| `profiling.include_field_mean_value` | | `True` | Whether to profile for the mean value of numeric columns. | +| `profiling.include_field_median_value` | | `True` | Whether to profile for the median value of numeric columns. | +| `profiling.include_field_stddev_value` | | `True` | Whether to profile for the standard deviation of numeric columns. | +| `profiling.include_field_quantiles` | | `True` | Whether to profile for the quantiles of numeric columns. | +| `profiling.include_field_distinct_value_frequencies` | | `False` | Whether to profile for distinct value frequencies. | +| `profiling.include_field_histogram` | | `False` | Whether to profile for the histogram for numeric fields. | +| `profiling.include_field_sample_values` | | `True` | Whether to profile for the sample values for all columns. | + + +## Valid path_spec.include + +```python +s3://my-bucket/foo/tests/bar.avro # single file table +s3://my-bucket/foo/tests/*.* # mulitple file level tables +s3://my-bucket/foo/tests/{table}/*.avro #table without partition +s3://my-bucket/foo/tests/{table}/*/*.avro #table where partitions are not specified +s3://my-bucket/foo/tests/{table}/*.* # table where no partitions as well as data type specified +s3://my-bucket/{dept}/tests/{table}/*.avro # specifying key wards to be used in display name +s3://my-bucket/{dept}/tests/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.avro # specify partition key and value format +s3://my-bucket/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.avro # specify partition value only format +s3://my-bucket/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # for all extensions +s3://my-bucket/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # table is present at 2 levels down in bucket +s3://my-bucket/*/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # table is present at 3 levels down in bucket +``` + +## Valid path_spec.exclude +- **/tests/** +- s3://my-bucket/hr/** +- **/tests/*.csv +- s3://my-bucket/foo/*/my_table/** +- +### Notes + +- {table} represents folder for which dataset will be created. +- include path must end with (*.* or *.[ext]) to represent leaf level. +- if *.[ext] is provided then only files with specified type will be scanned. +- /*/ represents single folder. +- {partition[i]} represents value of partition. +- {partition_key[i]} represents name of the partition. +- While extracting, “i” will be used to match partition_key to partition. +- all folder levels need to be specified in include. Only exclude path can have ** like matching. +- exclude path cannot have named variables ( {} ). +- Folder names should not contain {, }, *, / in their names. +- {folder} is reserved for internal working. please do not use in named variables. + + + +If you would like to write a more complicated function for resolving file names, then a [transformer](../transformers.md) would be a good fit. + +:::caution + +Specify as long fixed prefix ( with out /*/ ) as possible in `path_spec.include`. This will reduce the scanning time and cost, specifically on AWS S3 + +::: + +:::caution + +Running profiling against many tables or over many rows can run up significant costs. +While we've done our best to limit the expensiveness of the queries the profiler runs, you +should be prudent about the set of tables profiling is enabled on or the frequency +of the profiling runs. + +::: + +:::caution + +If you are ingesting datasets from AWS S3, we recommend running the ingestion on a server in the same region to avoid high egress costs. + +::: + +## Compatibility + +Profiles are computed with PyDeequ, which relies on PySpark. Therefore, for computing profiles, we currently require Spark 3.0.3 with Hadoop 3.2 to be installed and the `SPARK_HOME` and `SPARK_VERSION` environment variables to be set. The Spark+Hadoop binary can be downloaded [here](https://www.apache.org/dyn/closer.lua/spark/spark-3.0.3/spark-3.0.3-bin-hadoop3.2.tgz). + +For an example guide on setting up PyDeequ on AWS, see [this guide](https://aws.amazon.com/blogs/big-data/testing-data-quality-at-scale-with-pydeequ/). + +## Questions + +If you've got any questions on configuring this source, feel free to ping us on [our Slack](https://slack.datahubproject.io/)! diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index 8a737f34c8a3d7..7aed2e29137492 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -57,6 +57,14 @@ class BigQueryDatasetKey(ProjectIdKey): dataset_id: str +class FolderKey(PlatformKey): + folder_abs_path: str + + +class S3BucketKey(PlatformKey): + bucket_name: str + + class DatahubKeyJSONEncoder(json.JSONEncoder): # overload method default @@ -158,7 +166,7 @@ def gen_containers( # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), aspectName="dataPlatformInstance", aspect=DataPlatformInstance( - platform=f"{make_data_platform_urn(container_key.platform)}" + platform=f"{make_data_platform_urn(container_key.platform)}", ), ) wu = MetadataWorkUnit( diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py index fe232aa47f93c4..1cc7389ab6eb3f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/s3_util.py @@ -19,6 +19,10 @@ def strip_s3_prefix(s3_uri: str) -> str: ) +def get_bucket_relative_path(s3_uri: str) -> str: + return "/".join(strip_s3_prefix(s3_uri).split("/")[1:]) + + def make_s3_urn(s3_uri: str, env: str) -> str: s3_name = strip_s3_prefix(s3_uri) @@ -33,3 +37,11 @@ def make_s3_urn(s3_uri: str, env: str) -> str: return f"urn:li:dataset:(urn:li:dataPlatform:s3,{name}_{extension},{env})" return f"urn:li:dataset:(urn:li:dataPlatform:s3,{s3_name},{env})" + + +def get_bucket_name(s3_uri: str) -> str: + if not is_s3_uri(s3_uri): + raise ValueError( + f"Not an S3 URI. Must start with one of the following prefixes: {str(S3_PREFIXES)}" + ) + return strip_s3_prefix(s3_uri).split("/")[0] diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/s3/__init__.py new file mode 100644 index 00000000000000..ca167bf287b7b3 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/__init__.py @@ -0,0 +1,717 @@ +import dataclasses +import logging +import os +import re +from collections import OrderedDict +from datetime import datetime +from math import log10 +from typing import Any, Dict, Iterable, List, Optional, Tuple + +import pydeequ +from pydeequ.analyzers import AnalyzerContext +from pyspark.conf import SparkConf +from pyspark.sql import SparkSession +from pyspark.sql.dataframe import DataFrame +from pyspark.sql.types import ( + ArrayType, + BinaryType, + BooleanType, + ByteType, + DateType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + LongType, + MapType, + NullType, + ShortType, + StringType, + StructField, + StructType, + TimestampType, +) +from pyspark.sql.utils import AnalysisException +from smart_open import open as smart_open + +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataset_urn_with_platform_instance, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import ( + FolderKey, + KeyType, + PlatformKey, + S3BucketKey, + add_dataset_to_container, + gen_containers, +) +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.aws.s3_util import ( + get_bucket_name, + get_bucket_relative_path, + strip_s3_prefix, +) +from datahub.ingestion.source.s3.config import DataLakeSourceConfig +from datahub.ingestion.source.s3.profiling import _SingleTableProfiler +from datahub.ingestion.source.s3.report import DataLakeSourceReport +from datahub.ingestion.source.schema_inference import avro, csv_tsv, json, parquet +from datahub.ingestion.transformer.add_dataset_browse_path import ( + AddDatasetBrowsePathConfig, + AddDatasetBrowsePathTransformer, +) +from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot +from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + BooleanTypeClass, + BytesTypeClass, + DateTypeClass, + NullTypeClass, + NumberTypeClass, + RecordTypeClass, + SchemaFieldDataType, + SchemaMetadata, + StringTypeClass, + TimeTypeClass, +) +from datahub.metadata.schema_classes import ( + ChangeTypeClass, + DatasetPropertiesClass, + MapTypeClass, + OtherSchemaClass, +) +from datahub.telemetry import stats, telemetry +from datahub.utilities.perf_timer import PerfTimer + +# hide annoying debug errors from py4j +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + +# for a list of all types, see https://spark.apache.org/docs/3.0.3/api/python/_modules/pyspark/sql/types.html +_field_type_mapping = { + NullType: NullTypeClass, + StringType: StringTypeClass, + BinaryType: BytesTypeClass, + BooleanType: BooleanTypeClass, + DateType: DateTypeClass, + TimestampType: TimeTypeClass, + DecimalType: NumberTypeClass, + DoubleType: NumberTypeClass, + FloatType: NumberTypeClass, + ByteType: BytesTypeClass, + IntegerType: NumberTypeClass, + LongType: NumberTypeClass, + ShortType: NumberTypeClass, + ArrayType: NullTypeClass, + MapType: MapTypeClass, + StructField: RecordTypeClass, + StructType: RecordTypeClass, +} + + +def get_column_type( + report: SourceReport, dataset_name: str, column_type: str +) -> SchemaFieldDataType: + """ + Maps known Spark types to datahub types + """ + TypeClass: Any = None + + for field_type, type_class in _field_type_mapping.items(): + if isinstance(column_type, field_type): + TypeClass = type_class + break + + # if still not found, report the warning + if TypeClass is None: + report.report_warning( + dataset_name, f"unable to map type {column_type} to metadata schema" + ) + TypeClass = NullTypeClass + + return SchemaFieldDataType(type=TypeClass()) + + +# config flags to emit telemetry for +config_options_to_report = [ + "platform", + "use_relative_path", + "ignore_dotfiles", +] + +# profiling flags to emit telemetry for +profiling_flags_to_report = [ + "profile_table_level_only", + "include_field_null_count", + "include_field_min_value", + "include_field_max_value", + "include_field_mean_value", + "include_field_median_value", + "include_field_stddev_value", + "include_field_quantiles", + "include_field_distinct_value_frequencies", + "include_field_histogram", + "include_field_sample_values", +] + +S3_PREFIXES = ("s3://", "s3n://", "s3a://") + + +# LOCAL_BROWSE_PATH_TRANSFORMER_CONFIG = AddDatasetBrowsePathConfig( +# path_templates=["/ENV/PLATFORMDATASET_PARTS"], replace_existing=True +# ) +# +# LOCAL_BROWSE_PATH_TRANSFORMER = AddDatasetBrowsePathTransformer( +# ctx=None, config=LOCAL_BROWSE_PATH_TRANSFORMER_CONFIG +# ) + + +@dataclasses.dataclass +class TableData: + disaply_name: str + is_s3: bool + full_path: str + partitions: Optional[OrderedDict] + timestamp: datetime + table_path: str + + +class S3Source(Source): + source_config: DataLakeSourceConfig + report: DataLakeSourceReport + profiling_times_taken: List[float] + processed_containers: List[str] + + def __init__(self, config: DataLakeSourceConfig, ctx: PipelineContext): + super().__init__(ctx) + self.source_config = config + self.report = DataLakeSourceReport() + self.profiling_times_taken = [] + + config_report = { + config_option: config.dict().get(config_option) + for config_option in config_options_to_report + } + config_report = {**config_report, "profiling_enabled": config.profiling.enabled} + + telemetry.telemetry_instance.ping( + "data_lake_config", + config_report, + ) + + if config.profiling.enabled: + telemetry.telemetry_instance.ping( + "data_lake_profiling_config", + { + config_flag: config.profiling.dict().get(config_flag) + for config_flag in profiling_flags_to_report + }, + ) + self.init_spark() + + def init_spark(self): + + conf = SparkConf() + + conf.set( + "spark.jars.packages", + ",".join( + [ + "org.apache.hadoop:hadoop-aws:3.0.3", + "org.apache.spark:spark-avro_2.12:3.0.3", + pydeequ.deequ_maven_coord, + ] + ), + ) + + if self.source_config.aws_config is not None: + + aws_access_key_id = self.source_config.aws_config.aws_access_key_id + aws_secret_access_key = self.source_config.aws_config.aws_secret_access_key + aws_session_token = self.source_config.aws_config.aws_session_token + + aws_provided_credentials = [ + aws_access_key_id, + aws_secret_access_key, + aws_session_token, + ] + + if any(x is not None for x in aws_provided_credentials): + + # see https://hadoop.apache.org/docs/r3.0.3/hadoop-aws/tools/hadoop-aws/index.html#Changing_Authentication_Providers + if all(x is not None for x in aws_provided_credentials): + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider", + ) + + else: + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider", + ) + + if aws_access_key_id is not None: + conf.set("spark.hadoop.fs.s3a.access.key", aws_access_key_id) + if aws_secret_access_key is not None: + conf.set( + "spark.hadoop.fs.s3a.secret.key", + aws_secret_access_key, + ) + if aws_session_token is not None: + conf.set( + "spark.hadoop.fs.s3a.session.token", + aws_session_token, + ) + else: + # if no explicit AWS config is provided, use a default AWS credentials provider + conf.set( + "spark.hadoop.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider", + ) + + conf.set("spark.jars.excludes", pydeequ.f2j_maven_coord) + conf.set("spark.driver.memory", self.source_config.spark_driver_memory) + + self.spark = SparkSession.builder.config(conf=conf).getOrCreate() + + @classmethod + def create(cls, config_dict, ctx): + config = DataLakeSourceConfig.parse_obj(config_dict) + + return cls(config, ctx) + + def read_file_spark(self, file: str, ext: str) -> Optional[DataFrame]: + + logger.debug(f"Opening file {file} for profiling in spark") + file = file.replace("s3://", "s3a://") + + telemetry.telemetry_instance.ping("data_lake_file", {"extension": ext}) + + if ext.endswith(".parquet"): + df = self.spark.read.parquet(file) + elif ext.endswith(".csv"): + # see https://sparkbyexamples.com/pyspark/pyspark-read-csv-file-into-dataframe + df = self.spark.read.csv( + file, + header="True", + inferSchema="True", + sep=",", + ignoreLeadingWhiteSpace=True, + ignoreTrailingWhiteSpace=True, + ) + elif ext.endswith(".tsv"): + df = self.spark.read.csv( + file, + header="True", + inferSchema="True", + sep="\t", + ignoreLeadingWhiteSpace=True, + ignoreTrailingWhiteSpace=True, + ) + elif ext.endswith(".json"): + df = self.spark.read.json(file) + elif ext.endswith(".avro"): + try: + df = self.spark.read.format("avro").load(file) + except AnalysisException: + self.report.report_warning( + file, + "To ingest avro files, please install the spark-avro package: https://mvnrepository.com/artifact/org.apache.spark/spark-avro_2.12/3.0.3", + ) + return None + + # TODO: add support for more file types + # elif file.endswith(".orc"): + # df = self.spark.read.orc(file) + else: + self.report.report_warning(file, f"file {file} has unsupported extension") + return None + logger.debug(f"dataframe read for file {file} with row count {df.count()}") + # replace periods in names because they break PyDeequ + # see https://mungingdata.com/pyspark/avoid-dots-periods-column-names/ + return df.toDF(*(c.replace(".", "_") for c in df.columns)) + + def create_emit_containers( + self, + container_key: KeyType, + name: str, + sub_types: List[str], + parent_container_key: Optional[PlatformKey] = None, + domain_urn: Optional[str] = None, + ) -> Iterable[MetadataWorkUnit]: + if container_key.guid() not in self.processed_containers: + container_wus = gen_containers( + container_key=container_key, + name=name, + sub_types=sub_types, + parent_container_key=parent_container_key, + domain_urn=domain_urn, + ) + self.processed_containers.append(container_key.guid()) + logger.debug(f"Creating container with key: {container_key}") + for wu in container_wus: + self.report.report_workunit(wu) + yield wu + + def create_container_hierarchy( + self, table_data: TableData, dataset_urn: str + ) -> Iterable[MetadataWorkUnit]: + logger.debug(f"Creating containers for {dataset_urn}") + base_full_path = table_data.table_path + parent_key = None + if table_data.is_s3: + bucket_name = get_bucket_name(table_data.full_path) + bucket_key = self.gen_bucket_key(bucket_name) + yield from self.create_emit_containers( + container_key=bucket_key, + name=bucket_name, + sub_types=["S3 bucket"], + parent_container_key=None, + ) + parent_key = bucket_key + base_full_path = get_bucket_relative_path(table_data.table_path) + + for folder in base_full_path[: base_full_path.rfind("/")].split("/"): + abs_path = folder + if parent_key: + if isinstance(parent_key, S3BucketKey): + prefix = parent_key.bucket_name + elif isinstance(parent_key, FolderKey): + prefix = parent_key.folder_abs_path + abs_path = prefix + "/" + folder + folder_key = self.gen_folder_key(abs_path) + yield from self.create_emit_containers( + container_key=folder_key, + name=folder, + sub_types=["Folder"], + parent_container_key=parent_key, + ) + parent_key = folder_key + if parent_key is None: + logger.warning( + f"Failed to associate Dataset ({dataset_urn}) with container" + ) + return + yield from add_dataset_to_container(parent_key, dataset_urn) + + def get_fields(self, table_data: TableData) -> List: + if table_data.is_s3: + if self.source_config.aws_config is None: + raise ValueError("AWS config is required for S3 file sources") + + s3_client = self.source_config.aws_config.get_s3_client() + + file = smart_open( + table_data.full_path, "rb", transport_params={"client": s3_client} + ) + else: + + file = open(table_data.full_path, "rb") + + fields = [] + + try: + if table_data.full_path.endswith(".parquet"): + fields = parquet.ParquetInferrer().infer_schema(file) + elif table_data.full_path.endswith(".csv"): + fields = csv_tsv.CsvInferrer( + max_rows=self.source_config.max_rows + ).infer_schema(file) + elif table_data.full_path.endswith(".tsv"): + fields = csv_tsv.TsvInferrer( + max_rows=self.source_config.max_rows + ).infer_schema(file) + elif table_data.full_path.endswith(".json"): + fields = json.JsonInferrer().infer_schema(file) + elif table_data.full_path.endswith(".avro"): + fields = avro.AvroInferrer().infer_schema(file) + else: + self.report.report_warning( + table_data.full_path, + f"file {table_data.full_path} has unsupported extension", + ) + file.close() + except Exception as e: + self.report.report_warning( + table_data.full_path, + f"could not infer schema for file {table_data.full_path}: {e}", + ) + file.close() + logger.debug(f"Extracted fields in schema: {fields}") + fields = sorted(fields, key=lambda f: f.fieldPath) + + return fields + + def get_table_profile( + self, table_data: TableData, dataset_urn: str + ) -> Iterable[MetadataWorkUnit]: + # read in the whole table with Spark for profiling + table = self.read_file_spark( + table_data.table_path, os.path.splitext(table_data.full_path)[1] + ) + + # if table is not readable, skip + if table is None: + self.report.report_warning( + table_data.disaply_name, + f"unable to read table {table_data.disaply_name} from file {table_data.full_path}", + ) + return + + with PerfTimer() as timer: + # init PySpark analysis object + logger.debug( + f"Profiling {table_data.full_path}: reading file and computing nulls+uniqueness {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}" + ) + table_profiler = _SingleTableProfiler( + table, + self.spark, + self.source_config.profiling, + self.report, + table_data.full_path, + ) + + logger.debug( + f"Profiling {table_data.full_path}: preparing profilers to run {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}" + ) + # instead of computing each profile individually, we run them all in a single analyzer.run() call + # we use a single call because the analyzer optimizes the number of calls to the underlying profiler + # since multiple profiles reuse computations, this saves a lot of time + table_profiler.prepare_table_profiles() + + # compute the profiles + logger.debug( + f"Profiling {table_data.full_path}: computing profiles {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}" + ) + analysis_result = table_profiler.analyzer.run() + analysis_metrics = AnalyzerContext.successMetricsAsDataFrame( + self.spark, analysis_result + ) + + logger.debug( + f"Profiling {table_data.full_path}: extracting profiles {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}" + ) + table_profiler.extract_table_profiles(analysis_metrics) + + time_taken = timer.elapsed_seconds() + + logger.info( + f"Finished profiling {table_data.full_path}; took {time_taken:.3f} seconds" + ) + + self.profiling_times_taken.append(time_taken) + + mcp = MetadataChangeProposalWrapper( + entityType="dataset", + entityUrn=dataset_urn, + changeType=ChangeTypeClass.UPSERT, + aspectName="datasetProfile", + aspect=table_profiler.profile, + ) + wu = MetadataWorkUnit( + id=f"profile-{self.source_config.platform}-{table_data.table_path}", mcp=mcp + ) + self.report.report_workunit(wu) + yield wu + + def ingest_table(self, table_data: TableData) -> Iterable[MetadataWorkUnit]: + + logger.info(f"Extracting table schema from file: {table_data.full_path}") + browse_path: str = ( + strip_s3_prefix(table_data.table_path) + if table_data.is_s3 + else table_data.table_path.strip("/") + ) + + data_platform_urn = make_data_platform_urn(self.source_config.platform) + logger.info(f"Creating dataset urn with name: {browse_path}") + dataset_urn = make_dataset_urn_with_platform_instance( + self.source_config.platform, + browse_path, + self.source_config.platform_instance, + self.source_config.env, + ) + + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[], + ) + + dataset_properties = DatasetPropertiesClass( + description="", + name=table_data.disaply_name, + customProperties={}, + ) + dataset_snapshot.aspects.append(dataset_properties) + + fields = self.get_fields(table_data) + schema_metadata = SchemaMetadata( + schemaName=table_data.disaply_name, + platform=data_platform_urn, + version=0, + hash="", + fields=fields, + platformSchema=OtherSchemaClass(rawSchema=""), + ) + dataset_snapshot.aspects.append(schema_metadata) + + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + wu = MetadataWorkUnit(id=table_data.table_path, mce=mce) + self.report.report_workunit(wu) + yield wu + + yield from self.create_container_hierarchy(table_data, dataset_urn) + + if self.source_config.profiling.enabled: + yield from self.get_table_profile(table_data, dataset_urn) + + def gen_bucket_key(self, name): + return S3BucketKey( + platform="s3", + instance=self.source_config.env + if self.source_config.platform_instance is None + else self.source_config.platform_instance, + bucket_name=name, + ) + + def gen_folder_key(self, abs_path): + return FolderKey( + platform=self.source_config.platform, + instance=self.source_config.env + if self.source_config.platform_instance is None + else self.source_config.platform_instance, + folder_abs_path=abs_path, + ) + + def get_prefix(self, relative_path: str) -> str: + index = re.search("[\*|\{]", relative_path) # noqa: W605 + if index: + return relative_path[: index.start()] + else: + return relative_path + + def extract_table_name(self, named_vars: dict) -> str: + if self.source_config.path_spec.table_name is None: + raise ValueError("path_spec.table_name is not set") + return self.source_config.path_spec.table_name.format_map(named_vars) + + def extract_table_data(self, path: str, timestamp: datetime) -> TableData: + + logger.debug(f"Getting table data for path: {path}") + parsed_vars = self.source_config.path_spec.get_named_vars(path) + table_data = None + if parsed_vars is None or "table" not in parsed_vars.named: + table_data = TableData( + disaply_name=os.path.basename(path), + is_s3=self.source_config.path_spec.is_s3(), + full_path=path, + partitions=None, + timestamp=timestamp, + table_path=path, + ) + else: + include = self.source_config.path_spec.include + depth = include.count("/", 0, include.find("{table}")) + table_path = ( + "/".join(path.split("/")[:depth]) + "/" + parsed_vars.named["table"] + ) + table_data = TableData( + disaply_name=self.extract_table_name(parsed_vars.named), + is_s3=self.source_config.path_spec.is_s3(), + full_path=path, + partitions=None, + timestamp=timestamp, + table_path=table_path, + ) + return table_data + + def s3_browser(self) -> Iterable[tuple]: + if self.source_config.aws_config is None: + raise ValueError("aws_config not set. Cannot browse s3") + s3 = self.source_config.aws_config.get_s3_resource() + bucket_name = get_bucket_name(self.source_config.path_spec.include) + logger.debug(f"Scanning bucket : {bucket_name}") + bucket = s3.Bucket(bucket_name) + prefix = self.get_prefix( + get_bucket_relative_path(self.source_config.path_spec.include) + ) + logger.debug(f"Scanning objects with prefix:{prefix}") + for obj in bucket.objects.filter(Prefix=prefix).page_size(1000): + s3_path = f"s3://{obj.bucket_name}/{obj.key}" + yield s3_path, obj.last_modified + + def local_browser(self) -> Iterable[tuple]: + prefix = self.get_prefix(self.source_config.path_spec.include) + if os.path.isfile(prefix): + logger.debug(f"Scanning single local file: {prefix}") + yield prefix, os.path.getmtime(prefix) + else: + logger.debug(f"Scanning files under local folder: {prefix}") + for root, dirs, files in os.walk(prefix): + for file in sorted(files): + full_path = os.path.join(root, file) + yield full_path, os.path.getmtime(full_path) + + def get_workunits(self) -> Iterable[MetadataWorkUnit]: + self.processed_containers = [] + with PerfTimer() as timer: + + file_browser = ( + self.s3_browser() + if self.source_config.path_spec.is_s3() + else self.local_browser() + ) + table_dict: Dict[str, TableData] = {} + for file, timestamp in file_browser: + if not self.source_config.path_spec.allowed(file): + continue + table_data = self.extract_table_data(file, timestamp) + d_table_data = table_dict.setdefault(table_data.table_path, table_data) + if d_table_data.timestamp < table_data.timestamp: + table_dict[table_data.table_path] = table_data + for guid, table_data in table_dict.items(): + yield from self.ingest_table(table_data) + + if not self.source_config.profiling.enabled: + return + + total_time_taken = timer.elapsed_seconds() + + logger.info( + f"Profiling {len(self.profiling_times_taken)} table(s) finished in {total_time_taken:.3f} seconds" + ) + + time_percentiles: Dict[str, float] = {} + + if len(self.profiling_times_taken) > 0: + percentiles = [50, 75, 95, 99] + percentile_values = stats.calculate_percentiles( + self.profiling_times_taken, percentiles + ) + + time_percentiles = { + f"table_time_taken_p{percentile}": 10 + ** int(log10(percentile_values[percentile] + 1)) + for percentile in percentiles + } + + telemetry.telemetry_instance.ping( + "data_lake_profiling_summary", + # bucket by taking floor of log of time taken + { + "total_time_taken": 10 ** int(log10(total_time_taken + 1)), + "count": 10 ** int(log10(len(self.profiling_times_taken) + 1)), + "platform": self.source_config.platform, + **time_percentiles, + }, + ) + + def get_report(self): + return self.report + + def close(self): + pass diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/config.py b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py new file mode 100644 index 00000000000000..6f23415d0502fd --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/config.py @@ -0,0 +1,174 @@ +import logging +import os +import re +from typing import Any, Dict, List, Optional, Union + +import parse +import pydantic +from wcmatch import pathlib + +from datahub.configuration.common import AllowDenyPattern, ConfigModel +from datahub.emitter.mce_builder import DEFAULT_ENV +from datahub.ingestion.source.aws.aws_common import AwsSourceConfig +from datahub.ingestion.source.aws.s3_util import is_s3_uri +from datahub.ingestion.source.s3.profiling import DataLakeProfilerConfig + +# hide annoying debug errors from py4j +logging.getLogger("py4j").setLevel(logging.ERROR) +logger: logging.Logger = logging.getLogger(__name__) + +SUPPORTED_FILE_TYPES = ["csv", "tsv", "json", "parquet", "avro"] + + +class PathSpec(ConfigModel): + class Config: + arbitrary_types_allowed = True + + include: str + exclude: Optional[List[str]] + file_types: List[str] = SUPPORTED_FILE_TYPES + table_name: Optional[str] + + # to be set internally + _parsable_include: str + _compiled_include: parse.Parser + _glob_include: str + _is_s3: bool + + def allowed(self, path: str) -> bool: + logger.debug(f"Checking file to inclusion: {path}") + if not pathlib.PurePath(path).globmatch( + self._glob_include, flags=pathlib.GLOBSTAR + ): + return False + logger.debug(f"{path} matched include ") + if self.exclude: + for exclude_path in self.exclude: + if pathlib.PurePath(path).globmatch( + exclude_path, flags=pathlib.GLOBSTAR + ): + return False + logger.debug(f"{path} is not excluded") + ext = os.path.splitext(path)[1].strip(".") + + if ext != "*" and ext not in self.file_types: + return False + + logger.debug(f"{path} had selected extension {ext}") + logger.debug(f"{path} allowed for dataset creation") + return True + + def is_s3(self): + return self._is_s3 + + @classmethod + def get_parsable_include(cls, include: str) -> str: + parsable_include = include + for i in range(parsable_include.count("*")): + parsable_include = parsable_include.replace("*", f"{{folder[{i}]}}", 1) + return parsable_include + + def get_named_vars(self, path: str) -> Union[None, parse.Result, parse.Match]: + return self._compiled_include.parse(path) + + @pydantic.root_validator() + def validate_path_spec(cls, values: Dict) -> Dict[str, Any]: + + if "**" in values["include"]: + raise ValueError("path_spec.include cannot contain '**'") + + if values.get("file_types") is None: + values["file_types"] = SUPPORTED_FILE_TYPES + else: + for file_type in values["file_types"]: + if file_type not in SUPPORTED_FILE_TYPES: + raise ValueError( + f"file type {file_type} not in supported file types. Please specify one from {SUPPORTED_FILE_TYPES}" + ) + + include_ext = os.path.splitext(values["include"])[1].strip(".") + if include_ext not in values["file_types"] and include_ext != "*": + raise ValueError( + f"file type specified ({include_ext}) in path_spec.include is not in specified file " + f'types. Please select one from {values.get("file_types")} or specify ".*" to allow all types' + ) + + values["_parsable_include"] = PathSpec.get_parsable_include(values["include"]) + logger.debug(f'Setting _parsable_include: {values.get("_parsable_include")}') + compiled_include_tmp = parse.compile(values["_parsable_include"]) + values["_compiled_include"] = compiled_include_tmp + logger.debug(f'Setting _compiled_include: {values["_compiled_include"]}') + values["_glob_include"] = re.sub( + "\{[^}]+\}", "*", values["include"] # noqa: W605 + ) + logger.debug(f'Setting _glob_include: {values.get("_glob_include")}') + + if values.get("table_name") is None: + if "{table}" in values["include"]: + values["table_name"] = "{table}" + else: + logger.debug(f"include fields: {compiled_include_tmp.named_fields}") + logger.debug( + f"table_name fields: {parse.compile(values['table_name']).named_fields}" + ) + if not all( + x in values["_compiled_include"].named_fields + for x in parse.compile(values["table_name"]).named_fields + ): + raise ValueError( + "Not all named variables used in path_spec.table_name are specified in " + "path_spec.include" + ) + + if values.get("exclude") is not None: + for exclude_path in values["exclude"]: + if len(parse.compile(exclude_path).named_fields) != 0: + raise ValueError( + "path_spec.exclude should not contain any named variables" + ) + + values["_is_s3"] = is_s3_uri(values["include"]) + logger.debug(f'Setting _is_s3: {values.get("_is_s3")}') + return values + + +class DataLakeSourceConfig(ConfigModel): + path_spec: PathSpec + env: str = DEFAULT_ENV + platform_instance: Optional[str] = None + platform: str = "" # overwritten by validator below + + aws_config: Optional[AwsSourceConfig] = None + + profile_patterns: AllowDenyPattern = AllowDenyPattern.allow_all() + profiling: DataLakeProfilerConfig = DataLakeProfilerConfig() + + spark_driver_memory: str = "4g" + + max_rows: int = 100 + + @pydantic.root_validator(pre=False) + def validate_platform(cls, values: Dict) -> Dict: + value = values.get("platform") + if value is not None and value != "": + return values + if values.get("path_spec") is not None and isinstance( + values.get("path_spec"), PathSpec + ): + if values["path_spec"].is_s3(): + values["platform"] = "s3" + else: + values["platform"] = "file" + logger.debug(f'Setting config "platform": {values.get("platform")}') + return values + else: + raise ValueError("path_spec is not valid. Cannot autodetect platform") + + @pydantic.root_validator() + def ensure_profiling_pattern_is_passed_to_profiling( + cls, values: Dict[str, Any] + ) -> Dict[str, Any]: + profiling = values.get("profiling") + if profiling is not None and profiling.enabled: + profiling.allow_deny_patterns = values["profile_patterns"] + return values diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py b/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py new file mode 100644 index 00000000000000..b2afdd97f1cbdb --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/profiling.py @@ -0,0 +1,532 @@ +import dataclasses +from math import log10 +from typing import Any, Dict, List, Optional + +import pydantic +from pandas import DataFrame +from pydeequ.analyzers import ( + AnalysisRunBuilder, + AnalysisRunner, + AnalyzerContext, + ApproxCountDistinct, + ApproxQuantile, + ApproxQuantiles, + Histogram, + Maximum, + Mean, + Minimum, + StandardDeviation, +) +from pyspark.sql import SparkSession +from pyspark.sql.functions import col, count, isnan, when +from pyspark.sql.types import DataType as SparkDataType +from pyspark.sql.types import ( + DateType, + DecimalType, + DoubleType, + FloatType, + IntegerType, + LongType, + NullType, + ShortType, + StringType, + TimestampType, +) + +from datahub.configuration.common import AllowDenyPattern, ConfigModel +from datahub.emitter.mce_builder import get_sys_time +from datahub.ingestion.source.profiling.common import ( + Cardinality, + _convert_to_cardinality, +) +from datahub.ingestion.source.s3.report import DataLakeSourceReport +from datahub.metadata.schema_classes import ( + DatasetFieldProfileClass, + DatasetProfileClass, + HistogramClass, + QuantileClass, + ValueFrequencyClass, +) +from datahub.telemetry import telemetry + +NUM_SAMPLE_ROWS = 20 +QUANTILES = [0.05, 0.25, 0.5, 0.75, 0.95] +MAX_HIST_BINS = 25 + + +def null_str(value: Any) -> Optional[str]: + # str() with a passthrough for None. + return str(value) if value is not None else None + + +class DataLakeProfilerConfig(ConfigModel): + enabled: bool = False + + # These settings will override the ones below. + profile_table_level_only: bool = False + + allow_deny_patterns: AllowDenyPattern = AllowDenyPattern.allow_all() + + max_number_of_fields_to_profile: Optional[pydantic.PositiveInt] = None + + include_field_null_count: bool = True + include_field_min_value: bool = True + include_field_max_value: bool = True + include_field_mean_value: bool = True + include_field_median_value: bool = True + include_field_stddev_value: bool = True + include_field_quantiles: bool = True + include_field_distinct_value_frequencies: bool = True + include_field_histogram: bool = True + include_field_sample_values: bool = True + + @pydantic.root_validator() + def ensure_field_level_settings_are_normalized( + cls: "DataLakeProfilerConfig", values: Dict[str, Any] + ) -> Dict[str, Any]: + max_num_fields_to_profile_key = "max_number_of_fields_to_profile" + table_level_profiling_only_key = "profile_table_level_only" + max_num_fields_to_profile = values.get(max_num_fields_to_profile_key) + if values.get(table_level_profiling_only_key): + all_field_level_metrics: List[str] = [ + "include_field_null_count", + "include_field_min_value", + "include_field_max_value", + "include_field_mean_value", + "include_field_median_value", + "include_field_stddev_value", + "include_field_quantiles", + "include_field_distinct_value_frequencies", + "include_field_histogram", + "include_field_sample_values", + ] + # Suppress all field-level metrics + for field_level_metric in all_field_level_metrics: + values[field_level_metric] = False + assert ( + max_num_fields_to_profile is None + ), f"{max_num_fields_to_profile_key} should be set to None" + + return values + + +@dataclasses.dataclass +class _SingleColumnSpec: + column: str + column_profile: DatasetFieldProfileClass + + # if the histogram is a list of value frequencies (discrete data) or bins (continuous data) + histogram_distinct: Optional[bool] = None + + type_: SparkDataType = NullType # type:ignore + + unique_count: Optional[int] = None + non_null_count: Optional[int] = None + cardinality: Optional[Cardinality] = None + + +class _SingleTableProfiler: + spark: SparkSession + dataframe: DataFrame + analyzer: AnalysisRunBuilder + column_specs: List[_SingleColumnSpec] + row_count: int + profiling_config: DataLakeProfilerConfig + file_path: str + columns_to_profile: List[str] + ignored_columns: List[str] + profile: DatasetProfileClass + report: DataLakeSourceReport + + def __init__( + self, + dataframe: DataFrame, + spark: SparkSession, + profiling_config: DataLakeProfilerConfig, + report: DataLakeSourceReport, + file_path: str, + ): + self.spark = spark + self.dataframe = dataframe + self.analyzer = AnalysisRunner(spark).onData(dataframe) + self.column_specs = [] + self.row_count = dataframe.count() + self.profiling_config = profiling_config + self.file_path = file_path + self.columns_to_profile = [] + self.ignored_columns = [] + self.profile = DatasetProfileClass(timestampMillis=get_sys_time()) + self.report = report + + self.profile.rowCount = self.row_count + self.profile.columnCount = len(dataframe.columns) + + column_types = {x.name: x.dataType for x in dataframe.schema.fields} + + if self.profiling_config.profile_table_level_only: + + return + + # get column distinct counts + for column in dataframe.columns: + + if not self.profiling_config.allow_deny_patterns.allowed(column): + self.ignored_columns.append(column) + continue + + self.columns_to_profile.append(column) + # Normal CountDistinct is ridiculously slow + self.analyzer.addAnalyzer(ApproxCountDistinct(column)) + + if self.profiling_config.max_number_of_fields_to_profile is not None: + if ( + len(self.columns_to_profile) + > self.profiling_config.max_number_of_fields_to_profile + ): + columns_being_dropped = self.columns_to_profile[ + self.profiling_config.max_number_of_fields_to_profile : + ] + self.columns_to_profile = self.columns_to_profile[ + : self.profiling_config.max_number_of_fields_to_profile + ] + + self.report.report_file_dropped( + f"The max_number_of_fields_to_profile={self.profiling_config.max_number_of_fields_to_profile} reached. Profile of columns {self.file_path}({', '.join(sorted(columns_being_dropped))})" + ) + + analysis_result = self.analyzer.run() + analysis_metrics = AnalyzerContext.successMetricsAsJson( + self.spark, analysis_result + ) + + # reshape distinct counts into dictionary + column_distinct_counts = { + x["instance"]: int(x["value"]) + for x in analysis_metrics + if x["name"] == "ApproxCountDistinct" + } + + select_numeric_null_counts = [ + count( + when( + isnan(c) | col(c).isNull(), + c, + ) + ).alias(c) + for c in self.columns_to_profile + if column_types[column] in [DoubleType, FloatType] + ] + + # PySpark doesn't support isnan() on non-float/double columns + select_nonnumeric_null_counts = [ + count( + when( + col(c).isNull(), + c, + ) + ).alias(c) + for c in self.columns_to_profile + if column_types[column] not in [DoubleType, FloatType] + ] + + null_counts = dataframe.select( + select_numeric_null_counts + select_nonnumeric_null_counts + ) + column_null_counts = null_counts.toPandas().T[0].to_dict() + column_null_fractions = { + c: column_null_counts[c] / self.row_count for c in self.columns_to_profile + } + column_nonnull_counts = { + c: self.row_count - column_null_counts[c] for c in self.columns_to_profile + } + + column_unique_proportions = { + c: ( + column_distinct_counts[c] / column_nonnull_counts[c] + if column_nonnull_counts[c] > 0 + else 0 + ) + for c in self.columns_to_profile + } + + if self.profiling_config.include_field_sample_values: + # take sample and convert to Pandas DataFrame + if self.row_count < NUM_SAMPLE_ROWS: + # if row count is less than number to sample, just take all rows + rdd_sample = dataframe.rdd.take(self.row_count) + else: + rdd_sample = dataframe.rdd.takeSample(False, NUM_SAMPLE_ROWS, seed=0) + + # init column specs with profiles + for column in self.columns_to_profile: + column_profile = DatasetFieldProfileClass(fieldPath=column) + + column_spec = _SingleColumnSpec(column, column_profile) + + column_profile.uniqueCount = column_distinct_counts.get(column) + column_profile.uniqueProportion = column_unique_proportions.get(column) + column_profile.nullCount = column_null_counts.get(column) + column_profile.nullProportion = column_null_fractions.get(column) + if self.profiling_config.include_field_sample_values: + column_profile.sampleValues = sorted( + [str(x[column]) for x in rdd_sample] + ) + + column_spec.type_ = column_types[column] + column_spec.cardinality = _convert_to_cardinality( + column_distinct_counts[column], + column_null_fractions[column], + ) + + self.column_specs.append(column_spec) + + def prep_min_value(self, column: str) -> None: + if self.profiling_config.include_field_min_value: + self.analyzer.addAnalyzer(Minimum(column)) + + def prep_max_value(self, column: str) -> None: + if self.profiling_config.include_field_max_value: + self.analyzer.addAnalyzer(Maximum(column)) + + def prep_mean_value(self, column: str) -> None: + if self.profiling_config.include_field_mean_value: + self.analyzer.addAnalyzer(Mean(column)) + + def prep_median_value(self, column: str) -> None: + if self.profiling_config.include_field_median_value: + self.analyzer.addAnalyzer(ApproxQuantile(column, 0.5)) + + def prep_stdev_value(self, column: str) -> None: + if self.profiling_config.include_field_stddev_value: + self.analyzer.addAnalyzer(StandardDeviation(column)) + + def prep_quantiles(self, column: str) -> None: + if self.profiling_config.include_field_quantiles: + self.analyzer.addAnalyzer(ApproxQuantiles(column, QUANTILES)) + + def prep_distinct_value_frequencies(self, column: str) -> None: + if self.profiling_config.include_field_distinct_value_frequencies: + self.analyzer.addAnalyzer(Histogram(column)) + + def prep_field_histogram(self, column: str) -> None: + if self.profiling_config.include_field_histogram: + self.analyzer.addAnalyzer(Histogram(column, maxDetailBins=MAX_HIST_BINS)) + + def prepare_table_profiles(self) -> None: + + row_count = self.row_count + + telemetry.telemetry_instance.ping( + "profile_data_lake_table", + # bucket by taking floor of log of the number of rows scanned + {"rows_profiled": 10 ** int(log10(row_count + 1))}, + ) + + # loop through the columns and add the analyzers + for column_spec in self.column_specs: + column = column_spec.column + column_profile = column_spec.column_profile + type_ = column_spec.type_ + cardinality = column_spec.cardinality + + non_null_count = column_spec.non_null_count + unique_count = column_spec.unique_count + + if ( + self.profiling_config.include_field_null_count + and non_null_count is not None + ): + null_count = row_count - non_null_count + assert null_count >= 0 + column_profile.nullCount = null_count + if row_count > 0: + column_profile.nullProportion = null_count / row_count + + if unique_count is not None: + column_profile.uniqueCount = unique_count + if non_null_count is not None and non_null_count > 0: + column_profile.uniqueProportion = unique_count / non_null_count + + if isinstance( + type_, + ( + DecimalType, + DoubleType, + FloatType, + IntegerType, + LongType, + ShortType, + ), + ): + if cardinality == Cardinality.UNIQUE: + pass + elif cardinality in [ + Cardinality.ONE, + Cardinality.TWO, + Cardinality.VERY_FEW, + Cardinality.FEW, + ]: + column_spec.histogram_distinct = True + self.prep_distinct_value_frequencies(column) + elif cardinality in [ + Cardinality.MANY, + Cardinality.VERY_MANY, + Cardinality.UNIQUE, + ]: + column_spec.histogram_distinct = False + self.prep_min_value(column) + self.prep_max_value(column) + self.prep_mean_value(column) + self.prep_median_value(column) + self.prep_stdev_value(column) + self.prep_quantiles(column) + self.prep_field_histogram(column) + else: # unknown cardinality - skip + pass + + elif isinstance(type_, StringType): + if cardinality in [ + Cardinality.ONE, + Cardinality.TWO, + Cardinality.VERY_FEW, + Cardinality.FEW, + ]: + column_spec.histogram_distinct = True + self.prep_distinct_value_frequencies( + column, + ) + + elif isinstance(type_, (DateType, TimestampType)): + self.prep_min_value(column) + self.prep_max_value(column) + + # FIXME: Re-add histogram once kl_divergence has been modified to support datetimes + + if cardinality in [ + Cardinality.ONE, + Cardinality.TWO, + Cardinality.VERY_FEW, + Cardinality.FEW, + ]: + self.prep_distinct_value_frequencies( + column, + ) + + def extract_table_profiles( + self, + analysis_metrics: DataFrame, + ) -> None: + self.profile.fieldProfiles = [] + + analysis_metrics = analysis_metrics.toPandas() + # DataFrame with following columns: + # entity: "Column" for column profile, "Table" for table profile + # instance: name of column being profiled. "*" for table profiles + # name: name of metric. Histogram metrics are formatted as "Histogram.." + # value: value of metric + + column_metrics = analysis_metrics[analysis_metrics["entity"] == "Column"] + + # resolve histogram types for grouping + column_metrics["kind"] = column_metrics["name"].apply( + lambda x: "Histogram" if x.startswith("Histogram.") else x + ) + + column_histogram_metrics = column_metrics[column_metrics["kind"] == "Histogram"] + column_nonhistogram_metrics = column_metrics[ + column_metrics["kind"] != "Histogram" + ] + + histogram_columns = set() + + if len(column_histogram_metrics) > 0: + + # we only want the absolute counts for each histogram for now + column_histogram_metrics = column_histogram_metrics[ + column_histogram_metrics["name"].apply( + lambda x: x.startswith("Histogram.abs.") + ) + ] + # get the histogram bins by chopping off the "Histogram.abs." prefix + column_histogram_metrics["bin"] = column_histogram_metrics["name"].apply( + lambda x: x[14:] + ) + + # reshape histogram counts for easier access + histogram_counts = column_histogram_metrics.set_index(["instance", "bin"])[ + "value" + ] + + histogram_columns = set(histogram_counts.index.get_level_values(0)) + + profiled_columns = set() + + if len(column_nonhistogram_metrics) > 0: + # reshape other metrics for easier access + nonhistogram_metrics = column_nonhistogram_metrics.set_index( + ["instance", "name"] + )["value"] + + profiled_columns = set(nonhistogram_metrics.index.get_level_values(0)) + # histogram_columns = set(histogram_counts.index.get_level_values(0)) + + for column_spec in self.column_specs: + column = column_spec.column + column_profile = column_spec.column_profile + + if column not in profiled_columns: + continue + + # convert to Dict so we can use .get + deequ_column_profile = nonhistogram_metrics.loc[column].to_dict() + + # uniqueCount, uniqueProportion, nullCount, nullProportion, sampleValues already set in TableWrapper + column_profile.min = null_str(deequ_column_profile.get("Minimum")) + column_profile.max = null_str(deequ_column_profile.get("Maximum")) + column_profile.mean = null_str(deequ_column_profile.get("Mean")) + column_profile.median = null_str( + deequ_column_profile.get("ApproxQuantiles-0.5") + ) + column_profile.stdev = null_str( + deequ_column_profile.get("StandardDeviation") + ) + if all( + deequ_column_profile.get(f"ApproxQuantiles-{quantile}") is not None + for quantile in QUANTILES + ): + column_profile.quantiles = [ + QuantileClass( + quantile=str(quantile), + value=str(deequ_column_profile[f"ApproxQuantiles-{quantile}"]), + ) + for quantile in QUANTILES + ] + + if column in histogram_columns: + + column_histogram = histogram_counts.loc[column] + # sort so output is deterministic + column_histogram = column_histogram.sort_index() + + if column_spec.histogram_distinct: + + column_profile.distinctValueFrequencies = [ + ValueFrequencyClass( + value=value, frequency=int(column_histogram.loc[value]) + ) + for value in column_histogram.index + ] + # sort so output is deterministic + column_profile.distinctValueFrequencies = sorted( + column_profile.distinctValueFrequencies, key=lambda x: x.value + ) + + else: + + column_profile.histogram = HistogramClass( + [str(x) for x in column_histogram.index], + [float(x) for x in column_histogram], + ) + + # append the column profile to the dataset profile + self.profile.fieldProfiles.append(column_profile) diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/report.py b/metadata-ingestion/src/datahub/ingestion/source/s3/report.py new file mode 100644 index 00000000000000..b1d9177f3cc691 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/report.py @@ -0,0 +1,17 @@ +import dataclasses +from dataclasses import field as dataclass_field +from typing import List + +from datahub.ingestion.api.source import SourceReport + + +@dataclasses.dataclass +class DataLakeSourceReport(SourceReport): + files_scanned = 0 + filtered: List[str] = dataclass_field(default_factory=list) + + def report_file_scanned(self) -> None: + self.files_scanned += 1 + + def report_file_dropped(self, file: str) -> None: + self.filtered.append(file) diff --git a/metadata-ingestion/tests/integration/s3/data_lake_mces_golden.json b/metadata-ingestion/tests/integration/s3/data_lake_mces_golden.json new file mode 100644 index 00000000000000..094dbd4c13cda8 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/data_lake_mces_golden.json @@ -0,0 +1,3030 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.NPS,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "2", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Br \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Ca \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Cl \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Cond (\u00b5S/cm)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "DO (mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "DOC [mg/L C]", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "F \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "K \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Lat (\u00b0N)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Long (\u00b0W)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Mg \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "NH3-N \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "NO3-N+NO2-N \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Na \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "PO4-P \n(mg P/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Park ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "SO4-S \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "SUVA, 254nm", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Sampling Date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Secchi Depth (m)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Site ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "TDN \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "TDP \n(mg P/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "UV Absorbance, 254nm", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Water Temp (\u00b0C)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "d18O", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "dD", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "field29", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pH", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.NPS,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 33, \"columnCount\": 14, \"fieldProfiles\": [{\"fieldPath\": \"Sampling Date\", \"uniqueCount\": 12, \"uniqueProportion\": 0.36363636363636365, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"(mg N/L)\\\"\", \"frequency\": 3}, {\"value\": \"(mg P/L)\\\"\", \"frequency\": 2}, {\"value\": \"(mg/L)\\\"\", \"frequency\": 8}, {\"value\": \"6/21/2013\", \"frequency\": 3}, {\"value\": \"6/22/2013\", \"frequency\": 2}, {\"value\": \"6/23/2013\", \"frequency\": 1}, {\"value\": \"6/26/2014\", \"frequency\": 4}, {\"value\": \"6/27/2013\", \"frequency\": 1}, {\"value\": \"8/6/2014\", \"frequency\": 1}, {\"value\": \"8/7/2014\", \"frequency\": 3}, {\"value\": \"8/8/2014\", \"frequency\": 1}, {\"value\": \"9/16/2013\", \"frequency\": 4}], \"sampleValues\": [\"(mg/L)\\\"\", \"8/6/2014\", \"8/7/2014\", \"8/7/2014\", \"(mg/L)\\\"\", \"(mg N/L)\\\"\", \"(mg/L)\\\"\", \"9/16/2013\", \"6/22/2013\", \"(mg P/L)\\\"\", \"6/21/2013\", \"9/16/2013\", \"6/26/2014\", \"6/27/2013\", \"6/23/2013\", \"(mg N/L)\\\"\", \"6/26/2014\", \"9/16/2013\", \"(mg P/L)\\\"\", \"9/16/2013\"]}, {\"fieldPath\": \"Site ID\", \"uniqueCount\": 34, \"uniqueProportion\": 1.0303030303030303, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Br \", \"frequency\": 1}, {\"value\": \"Ca \", \"frequency\": 1}, {\"value\": \"Cl \", \"frequency\": 1}, {\"value\": \"Desperation Lake\", \"frequency\": 1}, {\"value\": \"Devil Mountain Lake\", \"frequency\": 1}, {\"value\": \"F \", \"frequency\": 1}, {\"value\": \"Feniak Lake\", \"frequency\": 1}, {\"value\": \"Imuruk Lake\", \"frequency\": 1}, {\"value\": \"Iniakuk Lake\", \"frequency\": 1}, {\"value\": \"K \", \"frequency\": 1}, {\"value\": \"Kurupa Lake\", \"frequency\": 1}, {\"value\": \"Kuzitrin Lake\", \"frequency\": 1}, {\"value\": \"Lake Kangilipak\", \"frequency\": 1}, {\"value\": \"Lake Matcharak\", \"frequency\": 1}, {\"value\": \"Lake Narvakrak\", \"frequency\": 1}, {\"value\": \"Lake Selby\", \"frequency\": 1}, {\"value\": \"Lava Lake\", \"frequency\": 1}, {\"value\": \"Mg \", \"frequency\": 1}, {\"value\": \"NO3-N+NO2-N \", \"frequency\": 1}, {\"value\": \"Na \", \"frequency\": 1}, {\"value\": \"North Killeak Lake\", \"frequency\": 1}, {\"value\": \"Nutavukti Lake\", \"frequency\": 1}, {\"value\": \"Okoklik Lake\", \"frequency\": 1}, {\"value\": \"PO4-P \", \"frequency\": 1}, {\"value\": \"SO4-S \", \"frequency\": 1}, {\"value\": \"Summit Lake\", \"frequency\": 1}, {\"value\": \"TDN \", \"frequency\": 1}, {\"value\": \"TDP \", \"frequency\": 1}, {\"value\": \"Takahula Lake\", \"frequency\": 1}, {\"value\": \"Walker Lake\", \"frequency\": 1}, {\"value\": \"White Fish Lake\", \"frequency\": 1}, {\"value\": \"Wild Lake\", \"frequency\": 1}, {\"value\": \"d18O\", \"frequency\": 1}], \"sampleValues\": [\"Br \", \"Kurupa Lake\", \"Lake Selby\", \"Nutavukti Lake\", \"Ca \", \"NO3-N+NO2-N \", \"SO4-S \", \"Lake Kangilipak\", \"Devil Mountain Lake\", \"TDN \", \"Imuruk Lake\", \"Feniak Lake\", \"Iniakuk Lake\", \"Lake Narvakrak\", \"North Killeak Lake\", \"TDP \", \"Lake Matcharak\", \"Okoklik Lake\", \"Cl \", \"Desperation Lake\"]}, {\"fieldPath\": \"Park ID\", \"uniqueCount\": 4, \"uniqueProportion\": 0.19047619047619047, \"nullCount\": 12, \"nullProportion\": 0.36363636363636365, \"distinctValueFrequencies\": [{\"value\": \"BELA\", \"frequency\": 6}, {\"value\": \"GAAR\", \"frequency\": 9}, {\"value\": \"NOAT\", \"frequency\": 5}, {\"value\": \"NullValue\", \"frequency\": 12}, {\"value\": \"dD\", \"frequency\": 1}], \"sampleValues\": [\"None\", \"GAAR\", \"GAAR\", \"GAAR\", \"None\", \"None\", \"None\", \"NOAT\", \"BELA\", \"None\", \"BELA\", \"NOAT\", \"GAAR\", \"NOAT\", \"BELA\", \"None\", \"GAAR\", \"NOAT\", \"None\", \"NOAT\"]}, {\"fieldPath\": \"Lat (\\ufffdN)\", \"uniqueCount\": 19, \"uniqueProportion\": 0.95, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"65.3845\", \"frequency\": 1}, {\"value\": \"65.58905\", \"frequency\": 1}, {\"value\": \"65.58996\", \"frequency\": 1}, {\"value\": \"66.32813\", \"frequency\": 1}, {\"value\": \"66.37863\", \"frequency\": 1}, {\"value\": \"66.40362\", \"frequency\": 1}, {\"value\": \"66.89298\", \"frequency\": 1}, {\"value\": \"67.01337\", \"frequency\": 1}, {\"value\": \"67.06375\", \"frequency\": 1}, {\"value\": \"67.1257\", \"frequency\": 1}, {\"value\": \"67.35014\", \"frequency\": 1}, {\"value\": \"67.50282\", \"frequency\": 1}, {\"value\": \"67.74715\", \"frequency\": 1}, {\"value\": \"67.9999\", \"frequency\": 1}, {\"value\": \"68.00064\", \"frequency\": 1}, {\"value\": \"68.01392\", \"frequency\": 1}, {\"value\": \"68.07008\", \"frequency\": 1}, {\"value\": \"68.24775\", \"frequency\": 1}, {\"value\": \"68.33031\", \"frequency\": 1}, {\"value\": \"68.35879\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"68.35879\", \"66.89298\", \"67.01337\", \"None\", \"None\", \"None\", \"68.00064\", \"66.40362\", \"None\", \"65.58905\", \"68.24775\", \"67.1257\", \"67.9999\", \"66.32813\", \"None\", \"67.74715\", \"68.01392\", \"None\", \"68.33031\"]}, {\"fieldPath\": \"Long (\\ufffdW)\", \"uniqueCount\": 20, \"uniqueProportion\": 1.0, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-158.31337\", \"frequency\": 1}, {\"value\": \"-158.74584\", \"frequency\": 1}, {\"value\": \"-159.15975\", \"frequency\": 1}, {\"value\": \"-159.27007\", \"frequency\": 1}, {\"value\": \"-161.72104\", \"frequency\": 1}, {\"value\": \"-163.18683\", \"frequency\": 1}, {\"value\": \"-163.21787\", \"frequency\": 1}, {\"value\": \"-163.90928\", \"frequency\": 1}, {\"value\": \"-164.10563\", \"frequency\": 1}, {\"value\": \"-164.48818\", \"frequency\": 1}, {\"value\": \"-164.7452\", \"frequency\": 1}, {\"value\": \"150.47092\", \"frequency\": 1}, {\"value\": \"151.57256\", \"frequency\": 1}, {\"value\": \"153.20834\", \"frequency\": 1}, {\"value\": \"153.66048\", \"frequency\": 1}, {\"value\": \"154.34215\", \"frequency\": 1}, {\"value\": \"154.60695\", \"frequency\": 1}, {\"value\": \"154.73245\", \"frequency\": 1}, {\"value\": \"155.65584\", \"frequency\": 1}, {\"value\": \"156.21262\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"154.60695\", \"155.65584\", \"154.73245\", \"None\", \"None\", \"None\", \"-159.15975\", \"-164.48818\", \"None\", \"-163.18683\", \"-158.31337\", \"153.20834\", \"-161.72104\", \"-164.10563\", \"None\", \"156.21262\", \"-159.27007\", \"None\", \"-158.74584\"]}, {\"fieldPath\": \"Water Temp (\\ufffdC)\", \"uniqueCount\": 16, \"uniqueProportion\": 0.8, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 4}, {\"value\": \"11.34\", \"frequency\": 1}, {\"value\": \"11.9\", \"frequency\": 1}, {\"value\": \"12.05\", \"frequency\": 1}, {\"value\": \"15.1\", \"frequency\": 1}, {\"value\": \"15.3\", \"frequency\": 1}, {\"value\": \"17.38\", \"frequency\": 1}, {\"value\": \"17.6\", \"frequency\": 1}, {\"value\": \"18.3\", \"frequency\": 1}, {\"value\": \"2.95\", \"frequency\": 1}, {\"value\": \"20.18\", \"frequency\": 1}, {\"value\": \"4.51\", \"frequency\": 1}, {\"value\": \"5.36\", \"frequency\": 1}, {\"value\": \"6.46\", \"frequency\": 2}, {\"value\": \"8.06\", \"frequency\": 1}, {\"value\": \"9.3\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"9.3\", \"15.1\", \"17.6\", \"None\", \"None\", \"None\", \"5.36\", \"6.46\", \"None\", \"17.38\", \"4.51\", \"-\", \"18.3\", \"11.34\", \"None\", \"-\", \"6.46\", \"None\", \"2.95\"]}, {\"fieldPath\": \"Cond (\\ufffdS/cm)\", \"uniqueCount\": 16, \"uniqueProportion\": 0.8, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 4}, {\"value\": \"107\", \"frequency\": 1}, {\"value\": \"108\", \"frequency\": 1}, {\"value\": \"129.2\", \"frequency\": 1}, {\"value\": \"132.4\", \"frequency\": 1}, {\"value\": \"1351\", \"frequency\": 1}, {\"value\": \"15\", \"frequency\": 1}, {\"value\": \"163.1\", \"frequency\": 1}, {\"value\": \"227.3\", \"frequency\": 1}, {\"value\": \"26\", \"frequency\": 1}, {\"value\": \"452\", \"frequency\": 1}, {\"value\": \"61\", \"frequency\": 2}, {\"value\": \"75\", \"frequency\": 1}, {\"value\": \"75.7\", \"frequency\": 1}, {\"value\": \"83\", \"frequency\": 1}, {\"value\": \"92\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"227.3\", \"163.1\", \"132.4\", \"None\", \"None\", \"None\", \"61\", \"107\", \"None\", \"26\", \"108\", \"-\", \"83\", \"1351\", \"None\", \"-\", \"75\", \"None\", \"61\"]}, {\"fieldPath\": \"pH\", \"uniqueCount\": 8, \"uniqueProportion\": 0.4, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 13}, {\"value\": \"6.44\", \"frequency\": 1}, {\"value\": \"7.31\", \"frequency\": 1}, {\"value\": \"7.42\", \"frequency\": 1}, {\"value\": \"7.45\", \"frequency\": 1}, {\"value\": \"7.69\", \"frequency\": 1}, {\"value\": \"7.82\", \"frequency\": 1}, {\"value\": \"8.04\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"-\", \"-\", \"-\", \"None\", \"None\", \"None\", \"-\", \"7.69\", \"None\", \"6.44\", \"-\", \"-\", \"7.31\", \"8.04\", \"None\", \"-\", \"-\", \"None\", \"-\"]}, {\"fieldPath\": \"DO (mg/L)\", \"uniqueCount\": 6, \"uniqueProportion\": 0.3157894736842105, \"nullCount\": 14, \"nullProportion\": 0.42424242424242425, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 14}, {\"value\": \"7.6\", \"frequency\": 1}, {\"value\": \"7.9\", \"frequency\": 1}, {\"value\": \"8.5\", \"frequency\": 1}, {\"value\": \"8.9\", \"frequency\": 1}, {\"value\": \"9.3\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 14}], \"sampleValues\": [\"None\", \"8.9\", \"8.5\", \"7.6\", \"None\", \"None\", \"None\", \"-\", \"-\", \"None\", \"-\", \"-\", \"-\", \"-\", \"-\", \"None\", \"None\", \"-\", \"None\", \"-\"]}, {\"fieldPath\": \"Secchi Depth (m)\", \"uniqueCount\": 4, \"uniqueProportion\": 0.2, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 16}, {\"value\": \"10\", \"frequency\": 2}, {\"value\": \"4.1\", \"frequency\": 1}, {\"value\": \"7\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"-\", \"-\", \"-\", \"None\", \"None\", \"None\", \"-\", \"-\", \"None\", \"-\", \"-\", \"7\", \"-\", \"-\", \"None\", \"10\", \"-\", \"None\", \"-\"]}, {\"fieldPath\": \"UV Absorbance, 254nm\", \"uniqueCount\": 20, \"uniqueProportion\": 1.0, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"0.02\", \"frequency\": 1}, {\"value\": \"0.044\", \"frequency\": 1}, {\"value\": \"0.048\", \"frequency\": 1}, {\"value\": \"0.059\", \"frequency\": 1}, {\"value\": \"0.06\", \"frequency\": 1}, {\"value\": \"0.062\", \"frequency\": 1}, {\"value\": \"0.063\", \"frequency\": 1}, {\"value\": \"0.076\", \"frequency\": 1}, {\"value\": \"0.091\", \"frequency\": 1}, {\"value\": \"0.095\", \"frequency\": 1}, {\"value\": \"0.104\", \"frequency\": 1}, {\"value\": \"0.107\", \"frequency\": 1}, {\"value\": \"0.117\", \"frequency\": 1}, {\"value\": \"0.151\", \"frequency\": 1}, {\"value\": \"0.154\", \"frequency\": 1}, {\"value\": \"0.186\", \"frequency\": 1}, {\"value\": \"0.191\", \"frequency\": 1}, {\"value\": \"0.212\", \"frequency\": 1}, {\"value\": \"0.223\", \"frequency\": 1}, {\"value\": \"0.436\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"0.044\", \"0.107\", \"0.117\", \"None\", \"None\", \"None\", \"0.212\", \"0.091\", \"None\", \"0.151\", \"0.048\", \"0.095\", \"0.154\", \"0.104\", \"None\", \"0.076\", \"0.191\", \"None\", \"0.062\"]}, {\"fieldPath\": \"DOC [mg/L C]\", \"uniqueCount\": 19, \"uniqueProportion\": 0.95, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"1.3\", \"frequency\": 1}, {\"value\": \"1.8\", \"frequency\": 1}, {\"value\": \"12.3\", \"frequency\": 1}, {\"value\": \"2.0\", \"frequency\": 1}, {\"value\": \"2.1\", \"frequency\": 2}, {\"value\": \"2.4\", \"frequency\": 1}, {\"value\": \"2.7\", \"frequency\": 1}, {\"value\": \"3.3\", \"frequency\": 1}, {\"value\": \"3.4\", \"frequency\": 1}, {\"value\": \"4.2\", \"frequency\": 1}, {\"value\": \"4.3\", \"frequency\": 1}, {\"value\": \"4.5\", \"frequency\": 1}, {\"value\": \"4.7\", \"frequency\": 1}, {\"value\": \"5.1\", \"frequency\": 1}, {\"value\": \"5.8\", \"frequency\": 1}, {\"value\": \"6.5\", \"frequency\": 1}, {\"value\": \"7.8\", \"frequency\": 1}, {\"value\": \"8.3\", \"frequency\": 1}, {\"value\": \"8.5\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"2.1\", \"4.2\", \"4.5\", \"None\", \"None\", \"None\", \"8.5\", \"3.4\", \"None\", \"4.7\", \"1.8\", \"3.3\", \"5.8\", \"4.3\", \"None\", \"5.1\", \"7.8\", \"None\", \"2.1\"]}, {\"fieldPath\": \"SUVA, 254nm\", \"uniqueCount\": 12, \"uniqueProportion\": 0.6, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"1.5\", \"frequency\": 2}, {\"value\": \"2.1\", \"frequency\": 1}, {\"value\": \"2.2\", \"frequency\": 2}, {\"value\": \"2.4\", \"frequency\": 2}, {\"value\": \"2.5\", \"frequency\": 2}, {\"value\": \"2.6\", \"frequency\": 3}, {\"value\": \"2.7\", \"frequency\": 2}, {\"value\": \"2.9\", \"frequency\": 2}, {\"value\": \"3.1\", \"frequency\": 1}, {\"value\": \"3.2\", \"frequency\": 1}, {\"value\": \"3.4\", \"frequency\": 1}, {\"value\": \"3.5\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"2.1\", \"2.5\", \"2.6\", \"None\", \"None\", \"None\", \"2.5\", \"2.6\", \"None\", \"3.2\", \"2.7\", \"2.9\", \"2.7\", \"2.4\", \"None\", \"1.5\", \"2.4\", \"None\", \"2.9\"]}, {\"fieldPath\": \"NH3-N \", \"uniqueCount\": 8, \"uniqueProportion\": 0.4, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"*0.001\", \"frequency\": 7}, {\"value\": \"*0.002\", \"frequency\": 4}, {\"value\": \"*0.003\", \"frequency\": 1}, {\"value\": \"*0.004\", \"frequency\": 2}, {\"value\": \"*0.007\", \"frequency\": 2}, {\"value\": \"*0.009\", \"frequency\": 1}, {\"value\": \"0.019\", \"frequency\": 1}, {\"value\": \"0.135\", \"frequency\": 2}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"None\", \"*0.001\", \"*0.002\", \"*0.003\", \"None\", \"None\", \"None\", \"*0.007\", \"*0.001\", \"None\", \"0.135\", \"*0.001\", \"*0.001\", \"*0.009\", \"*0.002\", \"None\", \"*0.001\", \"*0.004\", \"None\", \"*0.001\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.pokemon_abilities_json,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "pokemon_abilities_json.json", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.short_effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "is_main_series", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.is_hidden", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.slot", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.pokemon_abilities_json,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 1, \"columnCount\": 9, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}], \"sampleValues\": [\"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 1}], \"sampleValues\": [\"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.small,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "small.csv", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "2nd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3rd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "4th chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Progression Quality", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.small,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 5, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 3, \"uniqueProportion\": 0.6, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"4\", \"frequency\": 1}, {\"value\": \"6\", \"frequency\": 1}], \"sampleValues\": [\"6\", \"4\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 3, \"uniqueProportion\": 0.6, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 2}, {\"value\": \"6\", \"frequency\": 2}], \"sampleValues\": [\"4\", \"4\", \"1\", \"6\", \"6\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 3, \"uniqueProportion\": 0.6, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"2\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 1}], \"sampleValues\": [\"1\", \"1\", \"1\", \"4\", \"2\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 2, \"uniqueProportion\": 0.4, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}, {\"value\": \"5\", \"frequency\": 4}], \"sampleValues\": [\"5\", \"5\", \"1\", \"5\", \"5\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 4, \"uniqueProportion\": 0.8, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}], \"sampleValues\": [\"Alternative\", \"Catchy\", \"Didgeridoo\", \"Dreadful\", \"Dreadful\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.wa_fn_usec_hr_employee_attrition_csv,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "wa_fn_usec_hr_employee_attrition_csv.csv", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "age", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "attrition", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "businesstravel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "dailyrate", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "department", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "distancefromhome", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "education", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "educationfield", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "employeecount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "environmentsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "gender", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobinvolvement", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "joblevel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobrole", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "maritalstatus", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "numcompaniesworked", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "over18", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "overtime", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "percentsalaryhike", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "performancerating", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "relationshipsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "standardhours", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "stockoptionlevel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "totalworkingyears", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "trainingtimeslastyear", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "worklifebalance", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearsatcompany", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearsincurrentrole", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearssincelastpromotion", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearswithcurrmanager", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_2.wa_fn_usec_hr_employee_attrition_csv,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 249, \"columnCount\": 31, \"fieldProfiles\": [{\"fieldPath\": \"age\", \"uniqueCount\": 39, \"uniqueProportion\": 0.1566265060240964, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"19\", \"frequency\": 4}, {\"value\": \"20\", \"frequency\": 1}, {\"value\": \"21\", \"frequency\": 1}, {\"value\": \"22\", \"frequency\": 5}, {\"value\": \"23\", \"frequency\": 1}, {\"value\": \"24\", \"frequency\": 4}, {\"value\": \"25\", \"frequency\": 3}, {\"value\": \"26\", \"frequency\": 4}, {\"value\": \"27\", \"frequency\": 9}, {\"value\": \"28\", \"frequency\": 4}, {\"value\": \"29\", \"frequency\": 8}, {\"value\": \"30\", \"frequency\": 16}, {\"value\": \"31\", \"frequency\": 9}, {\"value\": \"32\", \"frequency\": 15}, {\"value\": \"33\", \"frequency\": 11}, {\"value\": \"34\", \"frequency\": 14}, {\"value\": \"35\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 13}, {\"value\": \"37\", \"frequency\": 11}, {\"value\": \"38\", \"frequency\": 11}, {\"value\": \"39\", \"frequency\": 3}, {\"value\": \"40\", \"frequency\": 7}, {\"value\": \"41\", \"frequency\": 9}, {\"value\": \"42\", \"frequency\": 3}, {\"value\": \"43\", \"frequency\": 5}, {\"value\": \"44\", \"frequency\": 4}, {\"value\": \"45\", \"frequency\": 8}, {\"value\": \"46\", \"frequency\": 7}, {\"value\": \"48\", \"frequency\": 1}, {\"value\": \"49\", \"frequency\": 3}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"51\", \"frequency\": 8}, {\"value\": \"52\", \"frequency\": 3}, {\"value\": \"53\", \"frequency\": 4}, {\"value\": \"54\", \"frequency\": 3}, {\"value\": \"55\", \"frequency\": 3}, {\"value\": \"56\", \"frequency\": 3}, {\"value\": \"57\", \"frequency\": 1}, {\"value\": \"58\", \"frequency\": 3}, {\"value\": \"59\", \"frequency\": 6}], \"sampleValues\": [\"34\", \"40\", \"44\", \"28\", \"38\", \"22\", \"58\", \"32\", \"32\", \"42\", \"27\", \"51\", \"50\", \"37\", \"41\", \"50\", \"24\", \"46\", \"52\", \"40\"]}, {\"fieldPath\": \"attrition\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"No\", \"frequency\": 209}, {\"value\": \"Yes\", \"frequency\": 40}], \"sampleValues\": [\"Yes\", \"No\", \"No\", \"Yes\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"Yes\", \"No\", \"No\", \"No\", \"No\", \"No\"]}, {\"fieldPath\": \"businesstravel\", \"uniqueCount\": 3, \"uniqueProportion\": 0.012048192771084338, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Non-Travel\", \"frequency\": 25}, {\"value\": \"Travel_Frequently\", \"frequency\": 46}, {\"value\": \"Travel_Rarely\", \"frequency\": 178}], \"sampleValues\": [\"Travel_Rarely\", \"Travel_Rarely\", \"Non-Travel\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Non-Travel\", \"Travel_Rarely\", \"Travel_Frequently\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Frequently\", \"Travel_Rarely\", \"Travel_Frequently\", \"Non-Travel\", \"Travel_Rarely\"]}, {\"fieldPath\": \"dailyrate\", \"uniqueCount\": 7, \"uniqueProportion\": 0.028112449799196786, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 49}, {\"value\": \"19\", \"frequency\": 17}, {\"value\": \"2\", \"frequency\": 22}, {\"value\": \"24\", \"frequency\": 11}, {\"value\": \"32\", \"frequency\": 15}, {\"value\": \"4\", \"frequency\": 15}, {\"value\": \"514\", \"frequency\": 24}, {\"value\": \"7\", \"frequency\": 96}], \"sampleValues\": [\"4\", \"0\", \"7\", \"2\", \"0\", \"7\", \"7\", \"32\", \"0\", \"0\", \"19\", \"7\", \"7\", \"32\", \"24\", \"7\", \"7\", \"7\", \"0\", \"7\"]}, {\"fieldPath\": \"department\", \"uniqueCount\": 3, \"uniqueProportion\": 0.012048192771084338, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Human Resources\", \"frequency\": 7}, {\"value\": \"Research & Development\", \"frequency\": 175}, {\"value\": \"Sales\", \"frequency\": 67}], \"sampleValues\": [\"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Sales\", \"Research & Development\", \"Sales\", \"Research & Development\", \"Sales\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Sales\", \"Research & Development\", \"Research & Development\", \"Sales\", \"Sales\"]}, {\"fieldPath\": \"distancefromhome\", \"uniqueCount\": 30, \"uniqueProportion\": 0.12048192771084337, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 42}, {\"value\": \"10\", \"frequency\": 4}, {\"value\": \"11\", \"frequency\": 6}, {\"value\": \"12\", \"frequency\": 3}, {\"value\": \"13\", \"frequency\": 1}, {\"value\": \"14\", \"frequency\": 5}, {\"value\": \"15\", \"frequency\": 3}, {\"value\": \"16\", \"frequency\": 4}, {\"value\": \"17\", \"frequency\": 1}, {\"value\": \"18\", \"frequency\": 6}, {\"value\": \"19\", \"frequency\": 5}, {\"value\": \"2\", \"frequency\": 29}, {\"value\": \"20\", \"frequency\": 4}, {\"value\": \"21\", \"frequency\": 5}, {\"value\": \"22\", \"frequency\": 2}, {\"value\": \"23\", \"frequency\": 8}, {\"value\": \"24\", \"frequency\": 4}, {\"value\": \"25\", \"frequency\": 3}, {\"value\": \"26\", \"frequency\": 6}, {\"value\": \"27\", \"frequency\": 3}, {\"value\": \"28\", \"frequency\": 2}, {\"value\": \"29\", \"frequency\": 4}, {\"value\": \"3\", \"frequency\": 16}, {\"value\": \"4\", \"frequency\": 12}, {\"value\": \"5\", \"frequency\": 14}, {\"value\": \"6\", \"frequency\": 18}, {\"value\": \"7\", \"frequency\": 11}, {\"value\": \"8\", \"frequency\": 11}, {\"value\": \"9\", \"frequency\": 17}], \"sampleValues\": [\"6\", \"19\", \"23\", \"24\", \"3\", \"2\", \"10\", \"1\", \"4\", \"4\", \"9\", \"6\", \"1\", \"1\", \"12\", \"12\", \"18\", \"1\", \"2\", \"4\"]}, {\"fieldPath\": \"education\", \"uniqueCount\": 5, \"uniqueProportion\": 0.020080321285140562, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 31}, {\"value\": \"2\", \"frequency\": 53}, {\"value\": \"3\", \"frequency\": 95}, {\"value\": \"4\", \"frequency\": 63}, {\"value\": \"5\", \"frequency\": 7}], \"sampleValues\": [\"1\", \"2\", \"3\", \"3\", \"3\", \"1\", \"4\", \"3\", \"2\", \"2\", \"3\", \"3\", \"3\", \"4\", \"3\", \"3\", \"1\", \"3\", \"4\", \"4\"]}, {\"fieldPath\": \"educationfield\", \"uniqueCount\": 6, \"uniqueProportion\": 0.024096385542168676, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Human Resources\", \"frequency\": 4}, {\"value\": \"Life Sciences\", \"frequency\": 102}, {\"value\": \"Marketing\", \"frequency\": 28}, {\"value\": \"Medical\", \"frequency\": 85}, {\"value\": \"Other\", \"frequency\": 11}, {\"value\": \"Technical Degree\", \"frequency\": 19}], \"sampleValues\": [\"Medical\", \"Medical\", \"Medical\", \"Life Sciences\", \"Life Sciences\", \"Technical Degree\", \"Medical\", \"Medical\", \"Life Sciences\", \"Technical Degree\", \"Medical\", \"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Technical Degree\", \"Marketing\", \"Life Sciences\", \"Medical\", \"Life Sciences\", \"Marketing\"]}, {\"fieldPath\": \"employeecount\", \"uniqueCount\": 1, \"uniqueProportion\": 0.004016064257028112, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 249}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"environmentsatisfaction\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 41}, {\"value\": \"2\", \"frequency\": 56}, {\"value\": \"3\", \"frequency\": 87}, {\"value\": \"4\", \"frequency\": 65}], \"sampleValues\": [\"2\", \"3\", \"2\", \"3\", \"1\", \"3\", \"4\", \"1\", \"3\", \"3\", \"4\", \"1\", \"1\", \"1\", \"2\", \"3\", \"2\", \"3\", \"1\", \"3\"]}, {\"fieldPath\": \"gender\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Female\", \"frequency\": 101}, {\"value\": \"Male\", \"frequency\": 148}], \"sampleValues\": [\"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Female\", \"Male\", \"Male\", \"Male\", \"Female\", \"Male\", \"Female\", \"Female\", \"Male\", \"Male\", \"Male\", \"Male\"]}, {\"fieldPath\": \"jobinvolvement\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 13}, {\"value\": \"2\", \"frequency\": 66}, {\"value\": \"3\", \"frequency\": 151}, {\"value\": \"4\", \"frequency\": 19}], \"sampleValues\": [\"3\", \"3\", \"3\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"2\", \"3\", \"3\", \"3\", \"2\", \"2\", \"2\"]}, {\"fieldPath\": \"joblevel\", \"uniqueCount\": 5, \"uniqueProportion\": 0.020080321285140562, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 101}, {\"value\": \"2\", \"frequency\": 78}, {\"value\": \"3\", \"frequency\": 45}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 15}], \"sampleValues\": [\"1\", \"2\", \"2\", \"1\", \"3\", \"1\", \"4\", \"2\", \"1\", \"5\", \"1\", \"5\", \"5\", \"2\", \"5\", \"3\", \"1\", \"3\", \"5\", \"3\"]}, {\"fieldPath\": \"jobrole\", \"uniqueCount\": 9, \"uniqueProportion\": 0.03614457831325301, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Healthcare Representative\", \"frequency\": 16}, {\"value\": \"Human Resources\", \"frequency\": 5}, {\"value\": \"Laboratory Technician\", \"frequency\": 51}, {\"value\": \"Manager\", \"frequency\": 19}, {\"value\": \"Manufacturing Director\", \"frequency\": 25}, {\"value\": \"Research Director\", \"frequency\": 13}, {\"value\": \"Research Scientist\", \"frequency\": 60}, {\"value\": \"Sales Executive\", \"frequency\": 48}, {\"value\": \"Sales Representative\", \"frequency\": 12}], \"sampleValues\": [\"Research Scientist\", \"Laboratory Technician\", \"Laboratory Technician\", \"Laboratory Technician\", \"Sales Executive\", \"Laboratory Technician\", \"Sales Executive\", \"Manufacturing Director\", \"Sales Representative\", \"Manager\", \"Research Scientist\", \"Research Director\", \"Research Director\", \"Manufacturing Director\", \"Research Director\", \"Sales Executive\", \"Laboratory Technician\", \"Healthcare Representative\", \"Manager\", \"Sales Executive\"]}, {\"fieldPath\": \"jobsatisfaction\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 46}, {\"value\": \"2\", \"frequency\": 43}, {\"value\": \"3\", \"frequency\": 74}, {\"value\": \"4\", \"frequency\": 86}], \"sampleValues\": [\"1\", \"4\", \"2\", \"3\", \"3\", \"4\", \"3\", \"4\", \"2\", \"4\", \"2\", \"3\", \"2\", \"3\", \"3\", \"4\", \"3\", \"1\", \"3\", \"4\"]}, {\"fieldPath\": \"maritalstatus\", \"uniqueCount\": 3, \"uniqueProportion\": 0.012048192771084338, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Divorced\", \"frequency\": 57}, {\"value\": \"Married\", \"frequency\": 105}, {\"value\": \"Single\", \"frequency\": 87}], \"sampleValues\": [\"Single\", \"Married\", \"Married\", \"Single\", \"Divorced\", \"Married\", \"Single\", \"Married\", \"Married\", \"Married\", \"Divorced\", \"Single\", \"Married\", \"Divorced\", \"Married\", \"Single\", \"Married\", \"Married\", \"Single\", \"Married\"]}, {\"fieldPath\": \"numcompaniesworked\", \"uniqueCount\": 10, \"uniqueProportion\": 0.040160642570281124, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 33}, {\"value\": \"1\", \"frequency\": 89}, {\"value\": \"2\", \"frequency\": 25}, {\"value\": \"3\", \"frequency\": 23}, {\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 9}, {\"value\": \"7\", \"frequency\": 13}, {\"value\": \"8\", \"frequency\": 8}, {\"value\": \"9\", \"frequency\": 10}], \"sampleValues\": [\"2\", \"8\", \"4\", \"5\", \"0\", \"0\", \"0\", \"1\", \"2\", \"1\", \"6\", \"7\", \"3\", \"1\", \"1\", \"4\", \"0\", \"2\", \"1\", \"7\"]}, {\"fieldPath\": \"over18\", \"uniqueCount\": 1, \"uniqueProportion\": 0.004016064257028112, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Y\", \"frequency\": 249}], \"sampleValues\": [\"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\"]}, {\"fieldPath\": \"overtime\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"No\", \"frequency\": 178}, {\"value\": \"Yes\", \"frequency\": 71}], \"sampleValues\": [\"No\", \"Yes\", \"No\", \"Yes\", \"No\", \"No\", \"No\", \"Yes\", \"No\", \"No\", \"No\", \"No\", \"Yes\", \"No\", \"No\", \"No\", \"No\", \"Yes\", \"Yes\", \"No\"]}, {\"fieldPath\": \"percentsalaryhike\", \"uniqueCount\": 15, \"uniqueProportion\": 0.060240963855421686, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"11\", \"frequency\": 32}, {\"value\": \"12\", \"frequency\": 32}, {\"value\": \"13\", \"frequency\": 34}, {\"value\": \"14\", \"frequency\": 35}, {\"value\": \"15\", \"frequency\": 16}, {\"value\": \"16\", \"frequency\": 13}, {\"value\": \"17\", \"frequency\": 13}, {\"value\": \"18\", \"frequency\": 16}, {\"value\": \"19\", \"frequency\": 10}, {\"value\": \"20\", \"frequency\": 9}, {\"value\": \"21\", \"frequency\": 9}, {\"value\": \"22\", \"frequency\": 17}, {\"value\": \"23\", \"frequency\": 9}, {\"value\": \"24\", \"frequency\": 2}, {\"value\": \"25\", \"frequency\": 2}], \"sampleValues\": [\"11\", \"15\", \"12\", \"14\", \"12\", \"14\", \"13\", \"22\", \"11\", \"11\", \"18\", \"13\", \"19\", \"18\", \"12\", \"11\", \"12\", \"13\", \"18\", \"11\"]}, {\"fieldPath\": \"performancerating\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"3\", \"frequency\": 201}, {\"value\": \"4\", \"frequency\": 48}], \"sampleValues\": [\"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\"]}, {\"fieldPath\": \"relationshipsatisfaction\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 47}, {\"value\": \"2\", \"frequency\": 32}, {\"value\": \"3\", \"frequency\": 81}, {\"value\": \"4\", \"frequency\": 89}], \"sampleValues\": [\"3\", \"3\", \"3\", \"2\", \"3\", \"3\", \"3\", \"2\", \"1\", \"4\", \"4\", \"3\", \"1\", \"3\", \"4\", \"4\", \"3\", \"3\", \"4\", \"1\"]}, {\"fieldPath\": \"standardhours\", \"uniqueCount\": 1, \"uniqueProportion\": 0.004016064257028112, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"80\", \"frequency\": 249}], \"sampleValues\": [\"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\"]}, {\"fieldPath\": \"stockoptionlevel\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 117}, {\"value\": \"1\", \"frequency\": 98}, {\"value\": \"2\", \"frequency\": 20}, {\"value\": \"3\", \"frequency\": 14}], \"sampleValues\": [\"0\", \"1\", \"1\", \"0\", \"2\", \"1\", \"0\", \"1\", \"1\", \"0\", \"1\", \"0\", \"0\", \"1\", \"0\", \"0\", \"1\", \"1\", \"0\", \"1\"]}, {\"fieldPath\": \"totalworkingyears\", \"uniqueCount\": 38, \"uniqueProportion\": 0.15261044176706828, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 2}, {\"value\": \"1\", \"frequency\": 14}, {\"value\": \"10\", \"frequency\": 27}, {\"value\": \"11\", \"frequency\": 6}, {\"value\": \"12\", \"frequency\": 13}, {\"value\": \"13\", \"frequency\": 7}, {\"value\": \"14\", \"frequency\": 4}, {\"value\": \"15\", \"frequency\": 3}, {\"value\": \"16\", \"frequency\": 11}, {\"value\": \"17\", \"frequency\": 12}, {\"value\": \"18\", \"frequency\": 2}, {\"value\": \"19\", \"frequency\": 3}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"20\", \"frequency\": 3}, {\"value\": \"21\", \"frequency\": 6}, {\"value\": \"22\", \"frequency\": 5}, {\"value\": \"23\", \"frequency\": 6}, {\"value\": \"24\", \"frequency\": 3}, {\"value\": \"25\", \"frequency\": 3}, {\"value\": \"26\", \"frequency\": 1}, {\"value\": \"28\", \"frequency\": 2}, {\"value\": \"29\", \"frequency\": 1}, {\"value\": \"3\", \"frequency\": 4}, {\"value\": \"30\", \"frequency\": 1}, {\"value\": \"31\", \"frequency\": 1}, {\"value\": \"32\", \"frequency\": 1}, {\"value\": \"33\", \"frequency\": 1}, {\"value\": \"34\", \"frequency\": 1}, {\"value\": \"36\", \"frequency\": 1}, {\"value\": \"37\", \"frequency\": 1}, {\"value\": \"38\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"40\", \"frequency\": 1}, {\"value\": \"5\", \"frequency\": 14}, {\"value\": \"6\", \"frequency\": 21}, {\"value\": \"7\", \"frequency\": 19}, {\"value\": \"8\", \"frequency\": 15}, {\"value\": \"9\", \"frequency\": 17}], \"sampleValues\": [\"8\", \"15\", \"17\", \"6\", \"17\", \"3\", \"38\", \"9\", \"6\", \"22\", \"6\", \"23\", \"28\", \"7\", \"23\", \"16\", \"6\", \"21\", \"33\", \"15\"]}, {\"fieldPath\": \"trainingtimeslastyear\", \"uniqueCount\": 7, \"uniqueProportion\": 0.028112449799196786, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 8}, {\"value\": \"1\", \"frequency\": 18}, {\"value\": \"2\", \"frequency\": 90}, {\"value\": \"3\", \"frequency\": 87}, {\"value\": \"4\", \"frequency\": 13}, {\"value\": \"5\", \"frequency\": 24}, {\"value\": \"6\", \"frequency\": 9}], \"sampleValues\": [\"2\", \"2\", \"3\", \"4\", \"6\", \"2\", \"1\", \"3\", \"5\", \"3\", \"1\", \"5\", \"1\", \"2\", \"0\", \"3\", \"2\", \"5\", \"2\", \"2\"]}, {\"fieldPath\": \"worklifebalance\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 64}, {\"value\": \"3\", \"frequency\": 146}, {\"value\": \"4\", \"frequency\": 25}], \"sampleValues\": [\"3\", \"4\", \"4\", \"3\", \"2\", \"3\", \"2\", \"3\", \"3\", \"3\", \"1\", \"3\", \"2\", \"4\", \"3\", \"3\", \"3\", \"2\", \"4\", \"2\"]}, {\"fieldPath\": \"yearsatcompany\", \"uniqueCount\": 29, \"uniqueProportion\": 0.11646586345381527, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 7}, {\"value\": \"1\", \"frequency\": 33}, {\"value\": \"10\", \"frequency\": 18}, {\"value\": \"11\", \"frequency\": 9}, {\"value\": \"12\", \"frequency\": 4}, {\"value\": \"13\", \"frequency\": 3}, {\"value\": \"14\", \"frequency\": 2}, {\"value\": \"15\", \"frequency\": 4}, {\"value\": \"16\", \"frequency\": 2}, {\"value\": \"17\", \"frequency\": 3}, {\"value\": \"19\", \"frequency\": 1}, {\"value\": \"2\", \"frequency\": 22}, {\"value\": \"20\", \"frequency\": 4}, {\"value\": \"21\", \"frequency\": 1}, {\"value\": \"22\", \"frequency\": 4}, {\"value\": \"24\", \"frequency\": 3}, {\"value\": \"25\", \"frequency\": 2}, {\"value\": \"27\", \"frequency\": 1}, {\"value\": \"3\", \"frequency\": 20}, {\"value\": \"33\", \"frequency\": 2}, {\"value\": \"37\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 19}, {\"value\": \"40\", \"frequency\": 1}, {\"value\": \"5\", \"frequency\": 31}, {\"value\": \"6\", \"frequency\": 11}, {\"value\": \"7\", \"frequency\": 15}, {\"value\": \"8\", \"frequency\": 9}, {\"value\": \"9\", \"frequency\": 17}], \"sampleValues\": [\"4\", \"7\", \"3\", \"4\", \"16\", \"2\", \"37\", \"9\", \"4\", \"22\", \"2\", \"20\", \"8\", \"7\", \"22\", \"2\", \"5\", \"10\", \"33\", \"12\"]}, {\"fieldPath\": \"yearsincurrentrole\", \"uniqueCount\": 19, \"uniqueProportion\": 0.07630522088353414, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 43}, {\"value\": \"1\", \"frequency\": 8}, {\"value\": \"10\", \"frequency\": 6}, {\"value\": \"11\", \"frequency\": 5}, {\"value\": \"12\", \"frequency\": 2}, {\"value\": \"13\", \"frequency\": 5}, {\"value\": \"14\", \"frequency\": 2}, {\"value\": \"15\", \"frequency\": 2}, {\"value\": \"16\", \"frequency\": 1}, {\"value\": \"17\", \"frequency\": 1}, {\"value\": \"18\", \"frequency\": 2}, {\"value\": \"2\", \"frequency\": 61}, {\"value\": \"3\", \"frequency\": 23}, {\"value\": \"4\", \"frequency\": 19}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 4}, {\"value\": \"7\", \"frequency\": 32}, {\"value\": \"8\", \"frequency\": 19}, {\"value\": \"9\", \"frequency\": 9}], \"sampleValues\": [\"2\", \"2\", \"2\", \"2\", \"10\", \"1\", \"10\", \"8\", \"3\", \"17\", \"2\", \"18\", \"3\", \"5\", \"15\", \"2\", \"3\", \"9\", \"7\", \"11\"]}, {\"fieldPath\": \"yearssincelastpromotion\", \"uniqueCount\": 15, \"uniqueProportion\": 0.060240963855421686, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 96}, {\"value\": \"1\", \"frequency\": 56}, {\"value\": \"10\", \"frequency\": 1}, {\"value\": \"11\", \"frequency\": 4}, {\"value\": \"12\", \"frequency\": 3}, {\"value\": \"13\", \"frequency\": 1}, {\"value\": \"15\", \"frequency\": 5}, {\"value\": \"2\", \"frequency\": 35}, {\"value\": \"3\", \"frequency\": 8}, {\"value\": \"4\", \"frequency\": 14}, {\"value\": \"5\", \"frequency\": 10}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 7}, {\"value\": \"8\", \"frequency\": 3}, {\"value\": \"9\", \"frequency\": 4}], \"sampleValues\": [\"1\", \"3\", \"1\", \"0\", \"5\", \"2\", \"1\", \"7\", \"1\", \"11\", \"2\", \"15\", \"0\", \"0\", \"15\", \"2\", \"1\", \"9\", \"15\", \"2\"]}, {\"fieldPath\": \"yearswithcurrmanager\", \"uniqueCount\": 16, \"uniqueProportion\": 0.0642570281124498, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 44}, {\"value\": \"1\", \"frequency\": 15}, {\"value\": \"10\", \"frequency\": 2}, {\"value\": \"11\", \"frequency\": 4}, {\"value\": \"12\", \"frequency\": 3}, {\"value\": \"13\", \"frequency\": 2}, {\"value\": \"15\", \"frequency\": 4}, {\"value\": \"17\", \"frequency\": 1}, {\"value\": \"2\", \"frequency\": 49}, {\"value\": \"3\", \"frequency\": 25}, {\"value\": \"4\", \"frequency\": 15}, {\"value\": \"5\", \"frequency\": 6}, {\"value\": \"6\", \"frequency\": 6}, {\"value\": \"7\", \"frequency\": 35}, {\"value\": \"8\", \"frequency\": 25}, {\"value\": \"9\", \"frequency\": 13}], \"sampleValues\": [\"3\", \"7\", \"2\", \"3\", \"13\", \"1\", \"8\", \"8\", \"2\", \"15\", \"2\", \"15\", \"7\", \"7\", \"8\", \"1\", \"2\", \"5\", \"12\", \"11\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.chord_progressions_avro,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_avro.avro", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=Root].[type=double].Progression Quality", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].1st chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].2nd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].3rd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=string].4th chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.chord_progressions_avro,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 28, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 5}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 7}, {\"value\": \"6\", \"frequency\": 5}], \"sampleValues\": [\"6\", \"4\", \"5\", \"1\", \"6\", \"4\", \"5\", \"4\", \"5\", \"4\", \"5\", \"1\", \"6\", \"1\", \"4\", \"3\", \"4\", \"4\", \"5\", \"1\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 7, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 10}, {\"value\": \"2\", \"frequency\": 2}, {\"value\": \"3\", \"frequency\": 2}, {\"value\": \"4\", \"frequency\": 6}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 1}], \"sampleValues\": [\"4\", \"1\", \"6\", \"4\", \"2\", \"1\", \"1\", \"1\", \"4\", \"5\", \"5\", \"4\", \"2\", \"1\", \"1\", \"4\", \"3\", \"5\", \"4\", \"5\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 6, \"uniqueProportion\": 0.21428571428571427, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"5\", \"frequency\": 8}, {\"value\": \"6\", \"frequency\": 5}, {\"value\": \"7\", \"frequency\": 2}], \"sampleValues\": [\"5\", \"5\", \"5\", \"5\", \"4\", \"5\", \"4\", \"1\", \"4\", \"5\", \"1\", \"6\", \"5\", \"1\", \"4\", \"6\", \"6\", \"4\", \"3\", \"7\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 20, \"uniqueProportion\": 0.7142857142857143, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}, {\"value\": \"Endless\", \"frequency\": 1}, {\"value\": \"Energetic\", \"frequency\": 1}, {\"value\": \"Flamenco\", \"frequency\": 2}, {\"value\": \"Folk\", \"frequency\": 2}, {\"value\": \"Grunge\", \"frequency\": 1}, {\"value\": \"Jazz\", \"frequency\": 1}, {\"value\": \"Love\", \"frequency\": 1}, {\"value\": \"Memories\", \"frequency\": 1}, {\"value\": \"Moody\", \"frequency\": 2}, {\"value\": \"Pop\", \"frequency\": 2}, {\"value\": \"Rebellious\", \"frequency\": 1}, {\"value\": \"Sad\", \"frequency\": 3}, {\"value\": \"Simple\", \"frequency\": 2}, {\"value\": \"Sweet\", \"frequency\": 1}, {\"value\": \"Wildside\", \"frequency\": 1}, {\"value\": \"Wistful\", \"frequency\": 1}], \"sampleValues\": [\"Dreadful\", \"Memories\", \"Flamenco\", \"Rebellious\", \"Endless\", \"Alternative\", \"Folk\", \"Sweet\", \"Sad\", \"Sad\", \"Simple\", \"Wistful\", \"Dreadful\", \"Didgeridoo\", \"Simple\", \"Energetic\", \"Grunge\", \"Sad\", \"Flamenco\", \"Moody\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 1, \"uniqueProportion\": 0.03571428571428571, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"NaN\", \"frequency\": 28}], \"sampleValues\": [\"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.chord_progressions_csv,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_csv.csv", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "2nd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3rd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "4th chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Progression Quality", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.chord_progressions_csv,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 28, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 19}, {\"value\": \"2\", \"frequency\": 3}, {\"value\": \"4\", \"frequency\": 2}, {\"value\": \"5\", \"frequency\": 1}, {\"value\": \"6\", \"frequency\": 3}], \"sampleValues\": [\"1\", \"1\", \"6\", \"4\", \"1\", \"6\", \"1\", \"5\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"6\", \"2\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 5}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 7}, {\"value\": \"6\", \"frequency\": 5}], \"sampleValues\": [\"6\", \"4\", \"5\", \"1\", \"6\", \"4\", \"5\", \"4\", \"5\", \"4\", \"5\", \"1\", \"6\", \"1\", \"4\", \"3\", \"4\", \"4\", \"5\", \"1\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 7, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 10}, {\"value\": \"2\", \"frequency\": 2}, {\"value\": \"3\", \"frequency\": 2}, {\"value\": \"4\", \"frequency\": 6}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 1}], \"sampleValues\": [\"4\", \"1\", \"6\", \"4\", \"2\", \"1\", \"1\", \"1\", \"4\", \"5\", \"5\", \"4\", \"2\", \"1\", \"1\", \"4\", \"3\", \"5\", \"4\", \"5\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 6, \"uniqueProportion\": 0.21428571428571427, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"5\", \"frequency\": 8}, {\"value\": \"6\", \"frequency\": 5}, {\"value\": \"7\", \"frequency\": 2}], \"sampleValues\": [\"5\", \"5\", \"5\", \"5\", \"4\", \"5\", \"4\", \"1\", \"4\", \"5\", \"1\", \"6\", \"5\", \"1\", \"4\", \"6\", \"6\", \"4\", \"3\", \"7\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 20, \"uniqueProportion\": 0.7142857142857143, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}, {\"value\": \"Endless\", \"frequency\": 1}, {\"value\": \"Energetic\", \"frequency\": 1}, {\"value\": \"Flamenco\", \"frequency\": 2}, {\"value\": \"Folk\", \"frequency\": 2}, {\"value\": \"Grunge\", \"frequency\": 1}, {\"value\": \"Jazz\", \"frequency\": 1}, {\"value\": \"Love\", \"frequency\": 1}, {\"value\": \"Memories\", \"frequency\": 1}, {\"value\": \"Moody\", \"frequency\": 2}, {\"value\": \"Pop\", \"frequency\": 2}, {\"value\": \"Rebellious\", \"frequency\": 1}, {\"value\": \"Sad\", \"frequency\": 3}, {\"value\": \"Simple\", \"frequency\": 2}, {\"value\": \"Sweet\", \"frequency\": 1}, {\"value\": \"Wildside\", \"frequency\": 1}, {\"value\": \"Wistful\", \"frequency\": 1}], \"sampleValues\": [\"Dreadful\", \"Memories\", \"Flamenco\", \"Rebellious\", \"Endless\", \"Alternative\", \"Folk\", \"Sweet\", \"Sad\", \"Sad\", \"Simple\", \"Wistful\", \"Dreadful\", \"Didgeridoo\", \"Simple\", \"Energetic\", \"Grunge\", \"Sad\", \"Flamenco\", \"Moody\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.countries_json,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "countries_json.json", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "countries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "countries.code", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "countries.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.countries_json,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 1, \"columnCount\": 1, \"fieldProfiles\": [{\"fieldPath\": \"countries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(code='DZ', name='Algeria'), Row(code='AR', name='Argentina'), Row(code='AU', name='Australia'), Row(code='AT', name='Austria'), Row(code=None, name='Austria-Hungary'), Row(code=None, name='Austrian Empire'), Row(code='AZ', name='Azerbaijan'), Row(code='BD', name='Bangladesh'), Row(code='BB', name='Barbados'), Row(code='DE', name='Bavaria'), Row(code='BY', name='Belarus'), Row(code='CD', name='Belgian Congo'), Row(code='BE', name='Belgium'), Row(code='BA', name='Bosnia'), Row(code='BA', name='Bosnia and Herzegovina'), Row(code='BR', name='Brazil'), Row(code='IN', name='British India'), Row(code='IL', name='British Mandate of Palestine'), Row(code='IL', name='British Protectorate of Palestine'), Row(code='LC', name='British West Indies'), Row(code='BG', name='Bulgaria'), Row(code='MM', name='Burma'), Row(code='CA', name='Canada'), Row(code='CL', name='Chile'), Row(code='CN', name='China'), Row(code='CO', name='Colombia'), Row(code='CR', name='Costa Rica'), Row(code='GR', name='Crete'), Row(code='HR', name='Croatia'), Row(code='CY', name='Cyprus'), Row(code='CZ', name='Czech Republic'), Row(code='CZ', name='Czechoslovakia'), Row(code='CD', name='Democratic Republic of the Congo'), Row(code='VN', name='Democratic Republic of Vietnam'), Row(code='DK', name='Denmark'), Row(code='DE', name='East Friesland'), Row(code='DE', name='East Germany'), Row(code='TL', name='East Timor'), Row(code='EG', name='Egypt'), Row(code='ET', name='Ethiopia'), Row(code='DK', name='Faroe Islands (Denmark)'), Row(code='FI', name='Finland'), Row(code='FR', name='France'), Row(code='PL', name='Free City of Danzig'), Row(code='DZ', name='French Algeria'), Row(code='GA', name='Gabon'), Row(code='PL', name='German-occupied Poland'), Row(code='DE', name='Germany'), Row(code='GH', name='Ghana'), Row(code='GH', name='Gold Coast'), Row(code='GR', name='Greece'), Row(code='GL', name='Greenland'), Row(code='GP', name='Guadeloupe Island'), Row(code='GT', name='Guatemala'), Row(code='DE', name='Hesse-Kassel'), Row(code='CZ', name='Hungarian Empire'), Row(code='HU', name='Hungary'), Row(code='IS', name='Iceland'), Row(code='IN', name='India'), Row(code='ID', name='Indonesia'), Row(code='IR', name='Iran'), Row(code='IQ', name='Iraq'), Row(code='IE', name='Ireland'), Row(code='IL', name='Israel'), Row(code='IT', name='Italy'), Row(code='JM', name='Jamaica'), Row(code='JP', name='Japan'), Row(code='ID', name='Java, Dutch East Indies'), Row(code='KE', name='Kenya'), Row(code='BR', name='Korea'), Row(code='LV', name='Latvia'), Row(code='LB', name='Lebanon'), Row(code='LR', name='Liberia'), Row(code='LT', name='Lithuania'), Row(code='LU', name='Luxembourg'), Row(code='MG', name='Madagascar'), Row(code='MU', name='Mauritius'), Row(code='DE', name='Mecklenburg'), Row(code='MX', name='Mexico'), Row(code='MA', name='Morocco'), Row(code='MM', name='Myanmar'), Row(code='NZ', name='New Zealand'), Row(code='NG', name='Nigeria'), Row(code='MK', name='North Macedonia'), Row(code='GB', name='Northern Ireland'), Row(code='ZM', name='Northern Rhodesia'), Row(code='NO', name='Norway'), Row(code='TR', name='Ottoman Empire'), Row(code='PK', name='Pakistan'), Row(code='US', name='Palestine'), Row(code='IR', name='Persia'), Row(code='PE', name='Peru'), Row(code='PH', name='Philippines'), Row(code='PL', name='Poland'), Row(code='PL', name='Poland, Russian Empire'), Row(code='PT', name='Portugal'), Row(code='PL', name='Prussia'), Row(code='PR', name='Puerto Rico'), Row(code='MK', name='Republic of Macedonia'), Row(code='RO', name='Romania'), Row(code='RU', name='Russia'), Row(code='RU', name='Russia, Soviet Union'), Row(code='AZ', name='Russian Empire'), Row(code='LC', name='Saint Lucia'), Row(code='DE', name='Schleswig'), Row(code='GB', name='Scotland'), Row(code='RS', name='Serbia'), Row(code='SG', name='Singapore'), Row(code='SK', name='Slovakia'), Row(code='SI', name='Slovenia'), Row(code='ZA', name='South Africa'), Row(code='KR', name='South Korea'), Row(code='ZW', name='Southern Rhodesia'), Row(code='ES', name='Spain'), Row(code='SE', name='Sweden'), Row(code='CH', name='Switzerland'), Row(code='TW', name='Taiwan'), Row(code='NL', name='the Netherlands'), Row(code='CN', name='Tibet'), Row(code='TT', name='Trinidad and Tobago'), Row(code='TN', name='Tunisia'), Row(code='TR', name='Turkey'), Row(code='IT', name='Tuscany'), Row(code='UA', name='Ukraine'), Row(code='ZA', name='Union of South Africa'), Row(code='GB', name='United Kingdom'), Row(code='US', name='USA'), Row(code='RU', name='USSR'), Row(code='VE', name='Venezuela'), Row(code='VN', name='Vietnam'), Row(code='DE', name='West Germany'), Row(code='DE', name='W\\u00fcrttemberg'), Row(code='YE', name='Yemen'), Row(code='YU', name='Yugoslavia'), Row(code='ZM', name='Zambia'), Row(code='ZW', name='Zimbabwe')]\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.food_avro,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_avro.avro", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=Root].[type=long].height", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].weight", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.food_avro,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 2, \"fieldProfiles\": [{\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"23\", \"50\", \"10\", \"50\", \"49\", \"43\", \"10\", \"23\", \"72\", \"72\", \"23\", \"10\", \"72\", \"50\", \"32\", \"32\", \"72\", \"49\", \"36\", \"43\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"6\", \"6\", \"4\", \"5\", \"5\", \"5\", \"4\", \"4\", \"6\", \"6\", \"7\", \"7\", \"4\", \"6\", \"4\", \"5\", \"6\", \"5\", \"4\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.food_csv,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_csv.csv", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.food_csv,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"cookie\", \"apple\", \"spinach\", \"apple\", \"lasagna\", \"orange\", \"spinach\", \"cookie\", \"pasta\", \"pasta\", \"cookie\", \"spinach\", \"pasta\", \"apple\", \"water\", \"water\", \"pasta\", \"lasagna\", \"chicken\", \"orange\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.12857142857142856, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"23\", \"50\", \"10\", \"50\", \"49\", \"43\", \"10\", \"23\", \"72\", \"72\", \"23\", \"10\", \"72\", \"50\", \"32\", \"32\", \"72\", \"49\", \"36\", \"43\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"6\", \"6\", \"4\", \"5\", \"5\", \"5\", \"4\", \"4\", \"6\", \"6\", \"7\", \"7\", \"4\", \"6\", \"4\", \"5\", \"6\", \"5\", \"4\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"brown\", \"red\", \"green\", \"red\", \"red\", \"orange\", \"green\", \"brown\", \"yellow\", \"yellow\", \"brown\", \"green\", \"yellow\", \"red\", \"blue\", \"blue\", \"yellow\", \"red\", \"white\", \"orange\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"True\", \"True\", \"True\", \"False\", \"True\", \"True\", \"False\", \"False\", \"False\", \"False\", \"True\", \"False\", \"None\", \"True\", \"True\", \"False\", \"False\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.food_parquet,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet.parquet", + "platform": "urn:li:dataPlatform:data-lake-test", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:data-lake-test,cases_1.food_parquet,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"cookie\", \"apple\", \"spinach\", \"apple\", \"lasagna\", \"orange\", \"spinach\", \"cookie\", \"pasta\", \"pasta\", \"cookie\", \"spinach\", \"pasta\", \"apple\", \"water\", \"water\", \"pasta\", \"lasagna\", \"chicken\", \"orange\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"23\", \"50\", \"10\", \"50\", \"49\", \"43\", \"10\", \"23\", \"72\", \"72\", \"23\", \"10\", \"72\", \"50\", \"32\", \"32\", \"72\", \"49\", \"36\", \"43\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"6\", \"6\", \"4\", \"5\", \"5\", \"5\", \"4\", \"4\", \"6\", \"6\", \"7\", \"7\", \"4\", \"6\", \"4\", \"5\", \"6\", \"5\", \"4\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"brown\", \"red\", \"green\", \"red\", \"red\", \"orange\", \"green\", \"brown\", \"yellow\", \"yellow\", \"brown\", \"green\", \"yellow\", \"red\", \"blue\", \"blue\", \"yellow\", \"red\", \"white\", \"orange\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"True\", \"True\", \"True\", \"False\", \"True\", \"True\", \"False\", \"False\", \"False\", \"False\", \"True\", \"False\", \"None\", \"True\", \"True\", \"False\", \"False\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "data-lake-test", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json new file mode 100644 index 00000000000000..84c290ac2168d6 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition.json @@ -0,0 +1,957 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:583fb3ef3a2b226ea2630157568eb7dc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bc816cf2df9acd90fcefa42dc425d886\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:d20e88ff88a6de6e53e437d342e218f4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:93525defb812252106d3b0c08a55e39a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:48a8653fc4afb55b12cd8d0280e09156\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:98a716614da5246426edd48260406364\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_csv", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06428571428571428, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json new file mode 100644 index 00000000000000..6df538a175f4a2 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_exclude.json @@ -0,0 +1,774 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:583fb3ef3a2b226ea2630157568eb7dc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bc816cf2df9acd90fcefa42dc425d886\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:d20e88ff88a6de6e53e437d342e218f4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:93525defb812252106d3b0c08a55e39a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:48a8653fc4afb55b12cd8d0280e09156\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:98a716614da5246426edd48260406364\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json new file mode 100644 index 00000000000000..fb99a059f955b9 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_filename.json @@ -0,0 +1,774 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "folder_aaa.food_csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:583fb3ef3a2b226ea2630157568eb7dc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bc816cf2df9acd90fcefa42dc425d886\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:d20e88ff88a6de6e53e437d342e218f4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:93525defb812252106d3b0c08a55e39a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:48a8653fc4afb55b12cd8d0280e09156\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:98a716614da5246426edd48260406364\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06428571428571428, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json new file mode 100644 index 00000000000000..240723b3070ba6 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_no_partition_glob.json @@ -0,0 +1,774 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:583fb3ef3a2b226ea2630157568eb7dc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bc816cf2df9acd90fcefa42dc425d886\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:d20e88ff88a6de6e53e437d342e218f4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:93525defb812252106d3b0c08a55e39a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:48a8653fc4afb55b12cd8d0280e09156\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:98a716614da5246426edd48260406364\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 140, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.06521739130434782, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 2}, {\"value\": \"apple\", \"frequency\": 14}, {\"value\": \"chicken\", \"frequency\": 14}, {\"value\": \"cookie\", \"frequency\": 12}, {\"value\": \"hamburger\", \"frequency\": 14}, {\"value\": \"lasagna\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"pasta\", \"frequency\": 14}, {\"value\": \"spinach\", \"frequency\": 14}, {\"value\": \"sushi\", \"frequency\": 14}, {\"value\": \"water\", \"frequency\": 14}], \"sampleValues\": [\"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"cookie\", \"hamburger\", \"lasagna\", \"lasagna\", \"lasagna\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"spinach\", \"sushi\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.07142857142857142, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 14}, {\"value\": \"23\", \"frequency\": 14}, {\"value\": \"32\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 14}, {\"value\": \"43\", \"frequency\": 14}, {\"value\": \"49\", \"frequency\": 14}, {\"value\": \"50\", \"frequency\": 14}, {\"value\": \"53\", \"frequency\": 14}, {\"value\": \"72\", \"frequency\": 14}], \"sampleValues\": [\"10\", \"10\", \"10\", \"10\", \"10\", \"2\", \"23\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"49\", \"49\", \"49\", \"50\", \"53\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.02857142857142857, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 48}, {\"value\": \"5\", \"frequency\": 30}, {\"value\": \"6\", \"frequency\": 46}, {\"value\": \"7\", \"frequency\": 16}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.05, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 14}, {\"value\": \"brown\", \"frequency\": 28}, {\"value\": \"green\", \"frequency\": 14}, {\"value\": \"orange\", \"frequency\": 28}, {\"value\": \"red\", \"frequency\": 28}, {\"value\": \"white\", \"frequency\": 14}, {\"value\": \"yellow\", \"frequency\": 14}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"green\", \"green\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.014492753623188406, \"nullCount\": 2, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json new file mode 100644 index 00000000000000..20ac7b9f7c71d2 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_basic.json @@ -0,0 +1,1381 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "folder_aaa.pokemon_abilities_json", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.short_effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "is_main_series", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.is_hidden", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.slot", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:583fb3ef3a2b226ea2630157568eb7dc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bc816cf2df9acd90fcefa42dc425d886\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:d20e88ff88a6de6e53e437d342e218f4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:93525defb812252106d3b0c08a55e39a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:48a8653fc4afb55b12cd8d0280e09156\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:98a716614da5246426edd48260406364\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 12, \"columnCount\": 11, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 12}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 12}], \"sampleValues\": [\"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}, {\"fieldPath\": \"year\", \"uniqueCount\": 3, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"2019\", \"frequency\": 4}, {\"value\": \"2020\", \"frequency\": 4}, {\"value\": \"2021\", \"frequency\": 4}], \"sampleValues\": [\"2019\", \"2019\", \"2019\", \"2019\", \"2020\", \"2020\", \"2020\", \"2020\", \"2021\", \"2021\", \"2021\", \"2021\"]}, {\"fieldPath\": \"month\", \"uniqueCount\": 4, \"uniqueProportion\": 0.3333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"april\", \"frequency\": 2}, {\"value\": \"feb\", \"frequency\": 4}, {\"value\": \"jan\", \"frequency\": 2}, {\"value\": \"march\", \"frequency\": 4}], \"sampleValues\": [\"april\", \"april\", \"feb\", \"feb\", \"feb\", \"feb\", \"jan\", \"jan\", \"march\", \"march\", \"march\", \"march\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json new file mode 100644 index 00000000000000..6dfd1a70643612 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_folder_partition_keyval.json @@ -0,0 +1,1381 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "folder_aaa.pokemon_abilities_json", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.short_effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "is_main_series", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.is_hidden", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.slot", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:583fb3ef3a2b226ea2630157568eb7dc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bc816cf2df9acd90fcefa42dc425d886", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:583fb3ef3a2b226ea2630157568eb7dc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d20e88ff88a6de6e53e437d342e218f4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:bc816cf2df9acd90fcefa42dc425d886\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:d20e88ff88a6de6e53e437d342e218f4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:93525defb812252106d3b0c08a55e39a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f6ff9cd64806a7bb00e2e3bf37acca50\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:48a8653fc4afb55b12cd8d0280e09156", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:93525defb812252106d3b0c08a55e39a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:98a716614da5246426edd48260406364", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:48a8653fc4afb55b12cd8d0280e09156\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"UAT\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:98a716614da5246426edd48260406364\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a0904d16a673fde8cbc8d0f2e167ecec\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 12, \"columnCount\": 11, \"fieldProfiles\": [{\"fieldPath\": \"effect_changes\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\", \"[Row(effect_entries=[Row(effect='Hat im Kampf keinen Effekt.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/')), Row(effect='Has no effect in battle.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'))], version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/'))]\"]}, {\"fieldPath\": \"effect_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\", \"[Row(effect='Attacken die Schaden verursachen haben mit jedem Treffer eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen, wenn die Attacke dies nicht bereits als Nebeneffekt hat.\\\\n\\\\nDer Effekt stapelt nicht mit dem von getragenen Items.\\\\n\\\\nAu\\u00dferhalb vom Kampf: Wenn ein Pok\\u00e9mon mit dieser F\\u00e4higkeit an erster Stelle im Team steht, tauchen wilde Pok\\u00e9mon nur halb so oft auf.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), short_effect='Mit jedem Treffer besteht eine 10% Chance das Ziel zur\\u00fcckschrecken zu lassen.'), Row(effect=\\\"This Pok\\u00e9mon's damaging moves have a 10% chance to make the target flinch with each hit if they do not already cause flinching as a secondary effect.\\\\n\\\\nThis ability does not stack with a held item.\\\\n\\\\nOverworld: The wild encounter rate is halved while this Pok\\u00e9mon is first in the party.\\\", language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), short_effect='Has a 10% chance of making target Pok\\u00e9mon flinch with each hit.')]\"]}, {\"fieldPath\": \"flavor_text_entries\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\", \"[Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ruby-sapphire', url='https://pokeapi.co/api/v2/version-group/5/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='emerald', url='https://pokeapi.co/api/v2/version-group/6/')), Row(flavor_text='Helps repel wild POK\\u00e9MON.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='firered-leafgreen', url='https://pokeapi.co/api/v2/version-group/7/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='diamond-pearl', url='https://pokeapi.co/api/v2/version-group/8/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='platinum', url='https://pokeapi.co/api/v2/version-group/9/')), Row(flavor_text='The stench helps keep\\\\nwild Pok\\u00e9mon away.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='heartgold-soulsilver', url='https://pokeapi.co/api/v2/version-group/10/')), Row(flavor_text='La puanteur peut\\\\neffrayer l\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-white', url='https://pokeapi.co/api/v2/version-group/11/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='black-2-white-2', url='https://pokeapi.co/api/v2/version-group/14/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='x-y', url='https://pokeapi.co/api/v2/version-group/15/')), Row(flavor_text='\\u304f\\u3055\\u304f\\u3066\\\\u3000\\u3042\\u3044\\u3066\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\uc545\\ucde8 \\ub54c\\ubb38\\uc5d0 \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='La puanteur peut effrayer\\\\nl\\u2019adversaire.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='L\\u00e4sst den Gegner durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='Es posible que el rival retroceda\\\\npor el mal olor.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='A volte il cattivo odore\\\\nfa tentennare i nemici.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='The stench may cause\\\\nthe target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u81ed\\u304f\\u3066\\\\u3000\\u76f8\\u624b\\u304c\\\\n\\u3072\\u308b\\u3080\\\\u3000\\u3053\\u3068\\u304c\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='omega-ruby-alpha-sapphire', url='https://pokeapi.co/api/v2/version-group/16/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='sun-moon', url='https://pokeapi.co/api/v2/version-group/17/')), Row(flavor_text='\\u304f\\u3055\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u306f\\u306a\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u3053\\u3046\\u3052\\u304d\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u3042\\u3044\\u3066\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\uc545\\ucde8\\ub97c \\ud48d\\uaca8\\uc11c\\\\n\\uacf5\\uaca9\\ud588\\uc744 \\ub54c \\uc0c1\\ub300\\uac00\\\\n\\ud480\\uc8fd\\uc744 \\ub54c\\uac00 \\uc788\\ub2e4.', language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u767c\\u51fa\\u81ed\\u6c23\\uff0c\\\\n\\u5728\\u653b\\u64ca\\u7684\\u6642\\u5019\\uff0c\\\\n\\u6709\\u6642\\u6703\\u4f7f\\u5c0d\\u624b\\u754f\\u7e2e\\u3002', language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Le Pok\\u00e9mon \\u00e9met une odeur si naus\\u00e9abonde\\\\nqu\\u2019il peut effrayer sa cible.', language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='L\\u00e4sst das Ziel beim Angriff eventuell durch Gestank\\\\nzur\\u00fcckschrecken.', language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='Debido al mal olor que emana, al atacar al rival puede\\\\nhacerlo retroceder.', language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='A volte il cattivo odore emesso dal Pok\\u00e9mon\\\\nfa tentennare i nemici quando attacca.', language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='By releasing stench when attacking, this Pok\\u00e9mon\\\\nmay cause the target to flinch.', language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u81ed\\u3044\\\\u3000\\u306b\\u304a\\u3044\\u3092\\\\u3000\\u653e\\u3064\\u3053\\u3068\\u306b\\u3088\\u3063\\u3066\\\\n\\u653b\\u6483\\u3057\\u305f\\\\u3000\\u3068\\u304d\\u306b\\\\u3000\\u76f8\\u624b\\u3092\\\\n\\u3072\\u308b\\u307e\\u305b\\u308b\\u3053\\u3068\\u304c\\\\u3000\\u3042\\u308b\\u3002', language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/')), Row(flavor_text='\\u901a\\u8fc7\\u91ca\\u653e\\u81ed\\u81ed\\u7684\\u6c14\\u5473\\uff0c\\\\n\\u5728\\u653b\\u51fb\\u7684\\u65f6\\u5019\\uff0c\\\\n\\u6709\\u65f6\\u4f1a\\u4f7f\\u5bf9\\u624b\\u754f\\u7f29\\u3002', language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), version_group=Row(name='ultra-sun-ultra-moon', url='https://pokeapi.co/api/v2/version-group/18/'))]\"]}, {\"fieldPath\": \"generation\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\", \"Row(name='generation-iii', url='https://pokeapi.co/api/v2/generation/3/')\"]}, {\"fieldPath\": \"id\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 12}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"is_main_series\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"stench\", \"frequency\": 12}], \"sampleValues\": [\"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\", \"stench\"]}, {\"fieldPath\": \"names\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\", \"[Row(language=Row(name='ja-Hrkt', url='https://pokeapi.co/api/v2/language/1/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='ko', url='https://pokeapi.co/api/v2/language/3/'), name='\\uc545\\ucde8'), Row(language=Row(name='zh-Hant', url='https://pokeapi.co/api/v2/language/4/'), name='\\u60e1\\u81ed'), Row(language=Row(name='fr', url='https://pokeapi.co/api/v2/language/5/'), name='Puanteur'), Row(language=Row(name='de', url='https://pokeapi.co/api/v2/language/6/'), name='Duftnote'), Row(language=Row(name='es', url='https://pokeapi.co/api/v2/language/7/'), name='Hedor'), Row(language=Row(name='it', url='https://pokeapi.co/api/v2/language/8/'), name='Tanfo'), Row(language=Row(name='en', url='https://pokeapi.co/api/v2/language/9/'), name='Stench'), Row(language=Row(name='ja', url='https://pokeapi.co/api/v2/language/11/'), name='\\u3042\\u304f\\u3057\\u3085\\u3046'), Row(language=Row(name='zh-Hans', url='https://pokeapi.co/api/v2/language/12/'), name='\\u6076\\u81ed')]\"]}, {\"fieldPath\": \"pokemon\", \"uniqueCount\": 1, \"uniqueProportion\": 0.08333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\", \"[Row(is_hidden=True, pokemon=Row(name='gloom', url='https://pokeapi.co/api/v2/pokemon/44/'), slot=3), Row(is_hidden=False, pokemon=Row(name='grimer', url='https://pokeapi.co/api/v2/pokemon/88/'), slot=1), Row(is_hidden=False, pokemon=Row(name='muk', url='https://pokeapi.co/api/v2/pokemon/89/'), slot=1), Row(is_hidden=False, pokemon=Row(name='stunky', url='https://pokeapi.co/api/v2/pokemon/434/'), slot=1), Row(is_hidden=False, pokemon=Row(name='skuntank', url='https://pokeapi.co/api/v2/pokemon/435/'), slot=1), Row(is_hidden=False, pokemon=Row(name='trubbish', url='https://pokeapi.co/api/v2/pokemon/568/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor', url='https://pokeapi.co/api/v2/pokemon/569/'), slot=1), Row(is_hidden=False, pokemon=Row(name='garbodor-gmax', url='https://pokeapi.co/api/v2/pokemon/10198/'), slot=1)]\"]}, {\"fieldPath\": \"year\", \"uniqueCount\": 3, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"2019\", \"frequency\": 4}, {\"value\": \"2020\", \"frequency\": 4}, {\"value\": \"2021\", \"frequency\": 4}], \"sampleValues\": [\"2019\", \"2019\", \"2019\", \"2019\", \"2020\", \"2020\", \"2020\", \"2020\", \"2021\", \"2021\", \"2021\", \"2021\"]}, {\"fieldPath\": \"month\", \"uniqueCount\": 4, \"uniqueProportion\": 0.3333333333333333, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"april\", \"frequency\": 2}, {\"value\": \"feb\", \"frequency\": 4}, {\"value\": \"jan\", \"frequency\": 2}, {\"value\": \"march\", \"frequency\": 4}], \"sampleValues\": [\"april\", \"april\", \"feb\", \"feb\", \"feb\", \"feb\", \"jan\", \"jan\", \"march\", \"march\", \"march\", \"march\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json new file mode 100644 index 00000000000000..e1b620e6b8da4d --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_multiple_files.json @@ -0,0 +1,2724 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "2", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Br \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Ca \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Cl \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Cond (\u00b5S/cm)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "DO (mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "DOC [mg/L C]", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "F \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "K \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Lat (\u00b0N)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Long (\u00b0W)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Mg \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "NH3-N \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "NO3-N+NO2-N \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Na \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "PO4-P \n(mg P/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Park ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "SO4-S \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "SUVA, 254nm", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Sampling Date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Secchi Depth (m)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Site ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "TDN \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "TDP \n(mg P/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "UV Absorbance, 254nm", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Water Temp (\u00b0C)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "d18O", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "dD", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "field29", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pH", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:30fb6a1dfbb1cf9c0ff92844b14f1e22", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:30fb6a1dfbb1cf9c0ff92844b14f1e22", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:30fb6a1dfbb1cf9c0ff92844b14f1e22", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2e8794cad300a557e34cec3fbfd48a15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2e8794cad300a557e34cec3fbfd48a15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2e8794cad300a557e34cec3fbfd48a15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2e8794cad300a557e34cec3fbfd48a15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:30fb6a1dfbb1cf9c0ff92844b14f1e22\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:9e02f1474054b9ad227be6b8ae5574a8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:9e02f1474054b9ad227be6b8ae5574a8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:9e02f1474054b9ad227be6b8ae5574a8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:9e02f1474054b9ad227be6b8ae5574a8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:2e8794cad300a557e34cec3fbfd48a15\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c0f8692822906bb838cb93bedf5cc860", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c0f8692822906bb838cb93bedf5cc860", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c0f8692822906bb838cb93bedf5cc860", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c0f8692822906bb838cb93bedf5cc860", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:9e02f1474054b9ad227be6b8ae5574a8\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:e05cdbb4122cad868f29eea7e9571346", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:e05cdbb4122cad868f29eea7e9571346", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:e05cdbb4122cad868f29eea7e9571346", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:e05cdbb4122cad868f29eea7e9571346", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:c0f8692822906bb838cb93bedf5cc860\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:161ffaf9bfa4603641b2fd53899edc52", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:161ffaf9bfa4603641b2fd53899edc52", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:161ffaf9bfa4603641b2fd53899edc52", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:161ffaf9bfa4603641b2fd53899edc52", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:e05cdbb4122cad868f29eea7e9571346\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:47d5326aae0dbc82c93eb7c2ad186bb4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:47d5326aae0dbc82c93eb7c2ad186bb4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:47d5326aae0dbc82c93eb7c2ad186bb4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:47d5326aae0dbc82c93eb7c2ad186bb4", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:161ffaf9bfa4603641b2fd53899edc52\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:42f9e89a8684547e92b91ba826cb751a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:42f9e89a8684547e92b91ba826cb751a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:42f9e89a8684547e92b91ba826cb751a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:42f9e89a8684547e92b91ba826cb751a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:47d5326aae0dbc82c93eb7c2ad186bb4\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 33, \"columnCount\": 14, \"fieldProfiles\": [{\"fieldPath\": \"Sampling Date\", \"uniqueCount\": 12, \"uniqueProportion\": 0.36363636363636365, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"(mg N/L)\\\"\", \"frequency\": 3}, {\"value\": \"(mg P/L)\\\"\", \"frequency\": 2}, {\"value\": \"(mg/L)\\\"\", \"frequency\": 8}, {\"value\": \"6/21/2013\", \"frequency\": 3}, {\"value\": \"6/22/2013\", \"frequency\": 2}, {\"value\": \"6/23/2013\", \"frequency\": 1}, {\"value\": \"6/26/2014\", \"frequency\": 4}, {\"value\": \"6/27/2013\", \"frequency\": 1}, {\"value\": \"8/6/2014\", \"frequency\": 1}, {\"value\": \"8/7/2014\", \"frequency\": 3}, {\"value\": \"8/8/2014\", \"frequency\": 1}, {\"value\": \"9/16/2013\", \"frequency\": 4}], \"sampleValues\": [\"(mg N/L)\\\"\", \"(mg N/L)\\\"\", \"(mg P/L)\\\"\", \"(mg P/L)\\\"\", \"(mg/L)\\\"\", \"(mg/L)\\\"\", \"(mg/L)\\\"\", \"6/21/2013\", \"6/22/2013\", \"6/23/2013\", \"6/26/2014\", \"6/26/2014\", \"6/27/2013\", \"8/6/2014\", \"8/7/2014\", \"8/7/2014\", \"9/16/2013\", \"9/16/2013\", \"9/16/2013\", \"9/16/2013\"]}, {\"fieldPath\": \"Site ID\", \"uniqueCount\": 34, \"uniqueProportion\": 1.0303030303030303, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Br \", \"frequency\": 1}, {\"value\": \"Ca \", \"frequency\": 1}, {\"value\": \"Cl \", \"frequency\": 1}, {\"value\": \"Desperation Lake\", \"frequency\": 1}, {\"value\": \"Devil Mountain Lake\", \"frequency\": 1}, {\"value\": \"F \", \"frequency\": 1}, {\"value\": \"Feniak Lake\", \"frequency\": 1}, {\"value\": \"Imuruk Lake\", \"frequency\": 1}, {\"value\": \"Iniakuk Lake\", \"frequency\": 1}, {\"value\": \"K \", \"frequency\": 1}, {\"value\": \"Kurupa Lake\", \"frequency\": 1}, {\"value\": \"Kuzitrin Lake\", \"frequency\": 1}, {\"value\": \"Lake Kangilipak\", \"frequency\": 1}, {\"value\": \"Lake Matcharak\", \"frequency\": 1}, {\"value\": \"Lake Narvakrak\", \"frequency\": 1}, {\"value\": \"Lake Selby\", \"frequency\": 1}, {\"value\": \"Lava Lake\", \"frequency\": 1}, {\"value\": \"Mg \", \"frequency\": 1}, {\"value\": \"NO3-N+NO2-N \", \"frequency\": 1}, {\"value\": \"Na \", \"frequency\": 1}, {\"value\": \"North Killeak Lake\", \"frequency\": 1}, {\"value\": \"Nutavukti Lake\", \"frequency\": 1}, {\"value\": \"Okoklik Lake\", \"frequency\": 1}, {\"value\": \"PO4-P \", \"frequency\": 1}, {\"value\": \"SO4-S \", \"frequency\": 1}, {\"value\": \"Summit Lake\", \"frequency\": 1}, {\"value\": \"TDN \", \"frequency\": 1}, {\"value\": \"TDP \", \"frequency\": 1}, {\"value\": \"Takahula Lake\", \"frequency\": 1}, {\"value\": \"Walker Lake\", \"frequency\": 1}, {\"value\": \"White Fish Lake\", \"frequency\": 1}, {\"value\": \"Wild Lake\", \"frequency\": 1}, {\"value\": \"d18O\", \"frequency\": 1}], \"sampleValues\": [\"Br \", \"Ca \", \"Cl \", \"Desperation Lake\", \"Devil Mountain Lake\", \"Feniak Lake\", \"Imuruk Lake\", \"Iniakuk Lake\", \"Kurupa Lake\", \"Lake Kangilipak\", \"Lake Matcharak\", \"Lake Narvakrak\", \"Lake Selby\", \"NO3-N+NO2-N \", \"North Killeak Lake\", \"Nutavukti Lake\", \"Okoklik Lake\", \"SO4-S \", \"TDN \", \"TDP \"]}, {\"fieldPath\": \"Park ID\", \"uniqueCount\": 4, \"uniqueProportion\": 0.19047619047619047, \"nullCount\": 12, \"nullProportion\": 0.36363636363636365, \"distinctValueFrequencies\": [{\"value\": \"BELA\", \"frequency\": 6}, {\"value\": \"GAAR\", \"frequency\": 9}, {\"value\": \"NOAT\", \"frequency\": 5}, {\"value\": \"NullValue\", \"frequency\": 12}, {\"value\": \"dD\", \"frequency\": 1}], \"sampleValues\": [\"BELA\", \"BELA\", \"BELA\", \"GAAR\", \"GAAR\", \"GAAR\", \"GAAR\", \"GAAR\", \"NOAT\", \"NOAT\", \"NOAT\", \"NOAT\", \"NOAT\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"Lat (\\ufffdN)\", \"uniqueCount\": 19, \"uniqueProportion\": 0.95, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"65.3845\", \"frequency\": 1}, {\"value\": \"65.58905\", \"frequency\": 1}, {\"value\": \"65.58996\", \"frequency\": 1}, {\"value\": \"66.32813\", \"frequency\": 1}, {\"value\": \"66.37863\", \"frequency\": 1}, {\"value\": \"66.40362\", \"frequency\": 1}, {\"value\": \"66.89298\", \"frequency\": 1}, {\"value\": \"67.01337\", \"frequency\": 1}, {\"value\": \"67.06375\", \"frequency\": 1}, {\"value\": \"67.1257\", \"frequency\": 1}, {\"value\": \"67.35014\", \"frequency\": 1}, {\"value\": \"67.50282\", \"frequency\": 1}, {\"value\": \"67.74715\", \"frequency\": 1}, {\"value\": \"67.9999\", \"frequency\": 1}, {\"value\": \"68.00064\", \"frequency\": 1}, {\"value\": \"68.01392\", \"frequency\": 1}, {\"value\": \"68.07008\", \"frequency\": 1}, {\"value\": \"68.24775\", \"frequency\": 1}, {\"value\": \"68.33031\", \"frequency\": 1}, {\"value\": \"68.35879\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"65.58905\", \"66.32813\", \"66.40362\", \"66.89298\", \"67.01337\", \"67.1257\", \"67.74715\", \"67.9999\", \"68.00064\", \"68.01392\", \"68.24775\", \"68.33031\", \"68.35879\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"Long (\\ufffdW)\", \"uniqueCount\": 20, \"uniqueProportion\": 1.0, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-158.31337\", \"frequency\": 1}, {\"value\": \"-158.74584\", \"frequency\": 1}, {\"value\": \"-159.15975\", \"frequency\": 1}, {\"value\": \"-159.27007\", \"frequency\": 1}, {\"value\": \"-161.72104\", \"frequency\": 1}, {\"value\": \"-163.18683\", \"frequency\": 1}, {\"value\": \"-163.21787\", \"frequency\": 1}, {\"value\": \"-163.90928\", \"frequency\": 1}, {\"value\": \"-164.10563\", \"frequency\": 1}, {\"value\": \"-164.48818\", \"frequency\": 1}, {\"value\": \"-164.7452\", \"frequency\": 1}, {\"value\": \"150.47092\", \"frequency\": 1}, {\"value\": \"151.57256\", \"frequency\": 1}, {\"value\": \"153.20834\", \"frequency\": 1}, {\"value\": \"153.66048\", \"frequency\": 1}, {\"value\": \"154.34215\", \"frequency\": 1}, {\"value\": \"154.60695\", \"frequency\": 1}, {\"value\": \"154.73245\", \"frequency\": 1}, {\"value\": \"155.65584\", \"frequency\": 1}, {\"value\": \"156.21262\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"-158.31337\", \"-158.74584\", \"-159.15975\", \"-159.27007\", \"-161.72104\", \"-163.18683\", \"-164.10563\", \"-164.48818\", \"153.20834\", \"154.60695\", \"154.73245\", \"155.65584\", \"156.21262\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"Water Temp (\\ufffdC)\", \"uniqueCount\": 16, \"uniqueProportion\": 0.8, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 4}, {\"value\": \"11.34\", \"frequency\": 1}, {\"value\": \"11.9\", \"frequency\": 1}, {\"value\": \"12.05\", \"frequency\": 1}, {\"value\": \"15.1\", \"frequency\": 1}, {\"value\": \"15.3\", \"frequency\": 1}, {\"value\": \"17.38\", \"frequency\": 1}, {\"value\": \"17.6\", \"frequency\": 1}, {\"value\": \"18.3\", \"frequency\": 1}, {\"value\": \"2.95\", \"frequency\": 1}, {\"value\": \"20.18\", \"frequency\": 1}, {\"value\": \"4.51\", \"frequency\": 1}, {\"value\": \"5.36\", \"frequency\": 1}, {\"value\": \"6.46\", \"frequency\": 2}, {\"value\": \"8.06\", \"frequency\": 1}, {\"value\": \"9.3\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"-\", \"-\", \"11.34\", \"15.1\", \"17.38\", \"17.6\", \"18.3\", \"2.95\", \"4.51\", \"5.36\", \"6.46\", \"6.46\", \"9.3\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"Cond (\\ufffdS/cm)\", \"uniqueCount\": 16, \"uniqueProportion\": 0.8, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 4}, {\"value\": \"107\", \"frequency\": 1}, {\"value\": \"108\", \"frequency\": 1}, {\"value\": \"129.2\", \"frequency\": 1}, {\"value\": \"132.4\", \"frequency\": 1}, {\"value\": \"1351\", \"frequency\": 1}, {\"value\": \"15\", \"frequency\": 1}, {\"value\": \"163.1\", \"frequency\": 1}, {\"value\": \"227.3\", \"frequency\": 1}, {\"value\": \"26\", \"frequency\": 1}, {\"value\": \"452\", \"frequency\": 1}, {\"value\": \"61\", \"frequency\": 2}, {\"value\": \"75\", \"frequency\": 1}, {\"value\": \"75.7\", \"frequency\": 1}, {\"value\": \"83\", \"frequency\": 1}, {\"value\": \"92\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"-\", \"-\", \"107\", \"108\", \"132.4\", \"1351\", \"163.1\", \"227.3\", \"26\", \"61\", \"61\", \"75\", \"83\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"pH\", \"uniqueCount\": 8, \"uniqueProportion\": 0.4, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 13}, {\"value\": \"6.44\", \"frequency\": 1}, {\"value\": \"7.31\", \"frequency\": 1}, {\"value\": \"7.42\", \"frequency\": 1}, {\"value\": \"7.45\", \"frequency\": 1}, {\"value\": \"7.69\", \"frequency\": 1}, {\"value\": \"7.82\", \"frequency\": 1}, {\"value\": \"8.04\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"6.44\", \"7.31\", \"7.69\", \"8.04\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"DO (mg/L)\", \"uniqueCount\": 6, \"uniqueProportion\": 0.3157894736842105, \"nullCount\": 14, \"nullProportion\": 0.42424242424242425, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 14}, {\"value\": \"7.6\", \"frequency\": 1}, {\"value\": \"7.9\", \"frequency\": 1}, {\"value\": \"8.5\", \"frequency\": 1}, {\"value\": \"8.9\", \"frequency\": 1}, {\"value\": \"9.3\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 14}], \"sampleValues\": [\"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"7.6\", \"8.5\", \"8.9\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"Secchi Depth (m)\", \"uniqueCount\": 4, \"uniqueProportion\": 0.2, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"-\", \"frequency\": 16}, {\"value\": \"10\", \"frequency\": 2}, {\"value\": \"4.1\", \"frequency\": 1}, {\"value\": \"7\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"-\", \"10\", \"7\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"UV Absorbance, 254nm\", \"uniqueCount\": 20, \"uniqueProportion\": 1.0, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"0.02\", \"frequency\": 1}, {\"value\": \"0.044\", \"frequency\": 1}, {\"value\": \"0.048\", \"frequency\": 1}, {\"value\": \"0.059\", \"frequency\": 1}, {\"value\": \"0.06\", \"frequency\": 1}, {\"value\": \"0.062\", \"frequency\": 1}, {\"value\": \"0.063\", \"frequency\": 1}, {\"value\": \"0.076\", \"frequency\": 1}, {\"value\": \"0.091\", \"frequency\": 1}, {\"value\": \"0.095\", \"frequency\": 1}, {\"value\": \"0.104\", \"frequency\": 1}, {\"value\": \"0.107\", \"frequency\": 1}, {\"value\": \"0.117\", \"frequency\": 1}, {\"value\": \"0.151\", \"frequency\": 1}, {\"value\": \"0.154\", \"frequency\": 1}, {\"value\": \"0.186\", \"frequency\": 1}, {\"value\": \"0.191\", \"frequency\": 1}, {\"value\": \"0.212\", \"frequency\": 1}, {\"value\": \"0.223\", \"frequency\": 1}, {\"value\": \"0.436\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"0.044\", \"0.048\", \"0.062\", \"0.076\", \"0.091\", \"0.095\", \"0.104\", \"0.107\", \"0.117\", \"0.151\", \"0.154\", \"0.191\", \"0.212\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"DOC [mg/L C]\", \"uniqueCount\": 19, \"uniqueProportion\": 0.95, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"1.3\", \"frequency\": 1}, {\"value\": \"1.8\", \"frequency\": 1}, {\"value\": \"12.3\", \"frequency\": 1}, {\"value\": \"2.0\", \"frequency\": 1}, {\"value\": \"2.1\", \"frequency\": 2}, {\"value\": \"2.4\", \"frequency\": 1}, {\"value\": \"2.7\", \"frequency\": 1}, {\"value\": \"3.3\", \"frequency\": 1}, {\"value\": \"3.4\", \"frequency\": 1}, {\"value\": \"4.2\", \"frequency\": 1}, {\"value\": \"4.3\", \"frequency\": 1}, {\"value\": \"4.5\", \"frequency\": 1}, {\"value\": \"4.7\", \"frequency\": 1}, {\"value\": \"5.1\", \"frequency\": 1}, {\"value\": \"5.8\", \"frequency\": 1}, {\"value\": \"6.5\", \"frequency\": 1}, {\"value\": \"7.8\", \"frequency\": 1}, {\"value\": \"8.3\", \"frequency\": 1}, {\"value\": \"8.5\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"1.8\", \"2.1\", \"2.1\", \"3.3\", \"3.4\", \"4.2\", \"4.3\", \"4.5\", \"4.7\", \"5.1\", \"5.8\", \"7.8\", \"8.5\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"SUVA, 254nm\", \"uniqueCount\": 12, \"uniqueProportion\": 0.6, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"1.5\", \"frequency\": 2}, {\"value\": \"2.1\", \"frequency\": 1}, {\"value\": \"2.2\", \"frequency\": 2}, {\"value\": \"2.4\", \"frequency\": 2}, {\"value\": \"2.5\", \"frequency\": 2}, {\"value\": \"2.6\", \"frequency\": 3}, {\"value\": \"2.7\", \"frequency\": 2}, {\"value\": \"2.9\", \"frequency\": 2}, {\"value\": \"3.1\", \"frequency\": 1}, {\"value\": \"3.2\", \"frequency\": 1}, {\"value\": \"3.4\", \"frequency\": 1}, {\"value\": \"3.5\", \"frequency\": 1}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"1.5\", \"2.1\", \"2.4\", \"2.4\", \"2.5\", \"2.5\", \"2.6\", \"2.6\", \"2.7\", \"2.7\", \"2.9\", \"2.9\", \"3.2\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}, {\"fieldPath\": \"NH3-N \", \"uniqueCount\": 8, \"uniqueProportion\": 0.4, \"nullCount\": 13, \"nullProportion\": 0.3939393939393939, \"distinctValueFrequencies\": [{\"value\": \"*0.001\", \"frequency\": 7}, {\"value\": \"*0.002\", \"frequency\": 4}, {\"value\": \"*0.003\", \"frequency\": 1}, {\"value\": \"*0.004\", \"frequency\": 2}, {\"value\": \"*0.007\", \"frequency\": 2}, {\"value\": \"*0.009\", \"frequency\": 1}, {\"value\": \"0.019\", \"frequency\": 1}, {\"value\": \"0.135\", \"frequency\": 2}, {\"value\": \"NullValue\", \"frequency\": 13}], \"sampleValues\": [\"*0.001\", \"*0.001\", \"*0.001\", \"*0.001\", \"*0.001\", \"*0.001\", \"*0.002\", \"*0.002\", \"*0.003\", \"*0.004\", \"*0.007\", \"*0.009\", \"0.135\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\", \"None\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "chord_progressions_avro.avro", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_avro.avro", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=Root].[type=double].Progression Quality", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].1st chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].2nd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].3rd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=string].4th chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 28, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 5}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 7}, {\"value\": \"6\", \"frequency\": 5}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 7, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 10}, {\"value\": \"2\", \"frequency\": 2}, {\"value\": \"3\", \"frequency\": 2}, {\"value\": \"4\", \"frequency\": 6}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 1}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"2\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 6, \"uniqueProportion\": 0.21428571428571427, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"5\", \"frequency\": 8}, {\"value\": \"6\", \"frequency\": 5}, {\"value\": \"7\", \"frequency\": 2}], \"sampleValues\": [\"1\", \"1\", \"1\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"7\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 20, \"uniqueProportion\": 0.7142857142857143, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}, {\"value\": \"Endless\", \"frequency\": 1}, {\"value\": \"Energetic\", \"frequency\": 1}, {\"value\": \"Flamenco\", \"frequency\": 2}, {\"value\": \"Folk\", \"frequency\": 2}, {\"value\": \"Grunge\", \"frequency\": 1}, {\"value\": \"Jazz\", \"frequency\": 1}, {\"value\": \"Love\", \"frequency\": 1}, {\"value\": \"Memories\", \"frequency\": 1}, {\"value\": \"Moody\", \"frequency\": 2}, {\"value\": \"Pop\", \"frequency\": 2}, {\"value\": \"Rebellious\", \"frequency\": 1}, {\"value\": \"Sad\", \"frequency\": 3}, {\"value\": \"Simple\", \"frequency\": 2}, {\"value\": \"Sweet\", \"frequency\": 1}, {\"value\": \"Wildside\", \"frequency\": 1}, {\"value\": \"Wistful\", \"frequency\": 1}], \"sampleValues\": [\"Alternative\", \"Didgeridoo\", \"Dreadful\", \"Dreadful\", \"Endless\", \"Energetic\", \"Flamenco\", \"Flamenco\", \"Folk\", \"Grunge\", \"Memories\", \"Moody\", \"Rebellious\", \"Sad\", \"Sad\", \"Sad\", \"Simple\", \"Simple\", \"Sweet\", \"Wistful\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 1, \"uniqueProportion\": 0.03571428571428571, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"NaN\", \"frequency\": 28}], \"sampleValues\": [\"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "chord_progressions_csv.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_csv.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "2nd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3rd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "4th chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Progression Quality", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 28, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 19}, {\"value\": \"2\", \"frequency\": 3}, {\"value\": \"4\", \"frequency\": 2}, {\"value\": \"5\", \"frequency\": 1}, {\"value\": \"6\", \"frequency\": 3}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"4\", \"5\", \"6\", \"6\", \"6\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 5}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 7}, {\"value\": \"6\", \"frequency\": 5}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 7, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 10}, {\"value\": \"2\", \"frequency\": 2}, {\"value\": \"3\", \"frequency\": 2}, {\"value\": \"4\", \"frequency\": 6}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 1}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"2\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 6, \"uniqueProportion\": 0.21428571428571427, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"5\", \"frequency\": 8}, {\"value\": \"6\", \"frequency\": 5}, {\"value\": \"7\", \"frequency\": 2}], \"sampleValues\": [\"1\", \"1\", \"1\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"7\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 20, \"uniqueProportion\": 0.7142857142857143, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}, {\"value\": \"Endless\", \"frequency\": 1}, {\"value\": \"Energetic\", \"frequency\": 1}, {\"value\": \"Flamenco\", \"frequency\": 2}, {\"value\": \"Folk\", \"frequency\": 2}, {\"value\": \"Grunge\", \"frequency\": 1}, {\"value\": \"Jazz\", \"frequency\": 1}, {\"value\": \"Love\", \"frequency\": 1}, {\"value\": \"Memories\", \"frequency\": 1}, {\"value\": \"Moody\", \"frequency\": 2}, {\"value\": \"Pop\", \"frequency\": 2}, {\"value\": \"Rebellious\", \"frequency\": 1}, {\"value\": \"Sad\", \"frequency\": 3}, {\"value\": \"Simple\", \"frequency\": 2}, {\"value\": \"Sweet\", \"frequency\": 1}, {\"value\": \"Wildside\", \"frequency\": 1}, {\"value\": \"Wistful\", \"frequency\": 1}], \"sampleValues\": [\"Alternative\", \"Didgeridoo\", \"Dreadful\", \"Dreadful\", \"Endless\", \"Energetic\", \"Flamenco\", \"Flamenco\", \"Folk\", \"Grunge\", \"Memories\", \"Moody\", \"Rebellious\", \"Sad\", \"Sad\", \"Sad\", \"Simple\", \"Simple\", \"Sweet\", \"Wistful\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "countries_json.json", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "countries_json.json", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "countries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "countries.code", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "countries.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 1, \"columnCount\": 1, \"fieldProfiles\": [{\"fieldPath\": \"countries\", \"uniqueCount\": 1, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"[Row(code='DZ', name='Algeria'), Row(code='AR', name='Argentina'), Row(code='AU', name='Australia'), Row(code='AT', name='Austria'), Row(code=None, name='Austria-Hungary'), Row(code=None, name='Austrian Empire'), Row(code='AZ', name='Azerbaijan'), Row(code='BD', name='Bangladesh'), Row(code='BB', name='Barbados'), Row(code='DE', name='Bavaria'), Row(code='BY', name='Belarus'), Row(code='CD', name='Belgian Congo'), Row(code='BE', name='Belgium'), Row(code='BA', name='Bosnia'), Row(code='BA', name='Bosnia and Herzegovina'), Row(code='BR', name='Brazil'), Row(code='IN', name='British India'), Row(code='IL', name='British Mandate of Palestine'), Row(code='IL', name='British Protectorate of Palestine'), Row(code='LC', name='British West Indies'), Row(code='BG', name='Bulgaria'), Row(code='MM', name='Burma'), Row(code='CA', name='Canada'), Row(code='CL', name='Chile'), Row(code='CN', name='China'), Row(code='CO', name='Colombia'), Row(code='CR', name='Costa Rica'), Row(code='GR', name='Crete'), Row(code='HR', name='Croatia'), Row(code='CY', name='Cyprus'), Row(code='CZ', name='Czech Republic'), Row(code='CZ', name='Czechoslovakia'), Row(code='CD', name='Democratic Republic of the Congo'), Row(code='VN', name='Democratic Republic of Vietnam'), Row(code='DK', name='Denmark'), Row(code='DE', name='East Friesland'), Row(code='DE', name='East Germany'), Row(code='TL', name='East Timor'), Row(code='EG', name='Egypt'), Row(code='ET', name='Ethiopia'), Row(code='DK', name='Faroe Islands (Denmark)'), Row(code='FI', name='Finland'), Row(code='FR', name='France'), Row(code='PL', name='Free City of Danzig'), Row(code='DZ', name='French Algeria'), Row(code='GA', name='Gabon'), Row(code='PL', name='German-occupied Poland'), Row(code='DE', name='Germany'), Row(code='GH', name='Ghana'), Row(code='GH', name='Gold Coast'), Row(code='GR', name='Greece'), Row(code='GL', name='Greenland'), Row(code='GP', name='Guadeloupe Island'), Row(code='GT', name='Guatemala'), Row(code='DE', name='Hesse-Kassel'), Row(code='CZ', name='Hungarian Empire'), Row(code='HU', name='Hungary'), Row(code='IS', name='Iceland'), Row(code='IN', name='India'), Row(code='ID', name='Indonesia'), Row(code='IR', name='Iran'), Row(code='IQ', name='Iraq'), Row(code='IE', name='Ireland'), Row(code='IL', name='Israel'), Row(code='IT', name='Italy'), Row(code='JM', name='Jamaica'), Row(code='JP', name='Japan'), Row(code='ID', name='Java, Dutch East Indies'), Row(code='KE', name='Kenya'), Row(code='BR', name='Korea'), Row(code='LV', name='Latvia'), Row(code='LB', name='Lebanon'), Row(code='LR', name='Liberia'), Row(code='LT', name='Lithuania'), Row(code='LU', name='Luxembourg'), Row(code='MG', name='Madagascar'), Row(code='MU', name='Mauritius'), Row(code='DE', name='Mecklenburg'), Row(code='MX', name='Mexico'), Row(code='MA', name='Morocco'), Row(code='MM', name='Myanmar'), Row(code='NZ', name='New Zealand'), Row(code='NG', name='Nigeria'), Row(code='MK', name='North Macedonia'), Row(code='GB', name='Northern Ireland'), Row(code='ZM', name='Northern Rhodesia'), Row(code='NO', name='Norway'), Row(code='TR', name='Ottoman Empire'), Row(code='PK', name='Pakistan'), Row(code='US', name='Palestine'), Row(code='IR', name='Persia'), Row(code='PE', name='Peru'), Row(code='PH', name='Philippines'), Row(code='PL', name='Poland'), Row(code='PL', name='Poland, Russian Empire'), Row(code='PT', name='Portugal'), Row(code='PL', name='Prussia'), Row(code='PR', name='Puerto Rico'), Row(code='MK', name='Republic of Macedonia'), Row(code='RO', name='Romania'), Row(code='RU', name='Russia'), Row(code='RU', name='Russia, Soviet Union'), Row(code='AZ', name='Russian Empire'), Row(code='LC', name='Saint Lucia'), Row(code='DE', name='Schleswig'), Row(code='GB', name='Scotland'), Row(code='RS', name='Serbia'), Row(code='SG', name='Singapore'), Row(code='SK', name='Slovakia'), Row(code='SI', name='Slovenia'), Row(code='ZA', name='South Africa'), Row(code='KR', name='South Korea'), Row(code='ZW', name='Southern Rhodesia'), Row(code='ES', name='Spain'), Row(code='SE', name='Sweden'), Row(code='CH', name='Switzerland'), Row(code='TW', name='Taiwan'), Row(code='NL', name='the Netherlands'), Row(code='CN', name='Tibet'), Row(code='TT', name='Trinidad and Tobago'), Row(code='TN', name='Tunisia'), Row(code='TR', name='Turkey'), Row(code='IT', name='Tuscany'), Row(code='UA', name='Ukraine'), Row(code='ZA', name='Union of South Africa'), Row(code='GB', name='United Kingdom'), Row(code='US', name='USA'), Row(code='RU', name='USSR'), Row(code='VE', name='Venezuela'), Row(code='VN', name='Vietnam'), Row(code='DE', name='West Germany'), Row(code='DE', name='W\\u00fcrttemberg'), Row(code='YE', name='Yemen'), Row(code='YU', name='Yugoslavia'), Row(code='ZM', name='Zambia'), Row(code='ZW', name='Zimbabwe')]\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet.parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet.parquet", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 70, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"name\", \"uniqueCount\": 9, \"uniqueProportion\": 0.13043478260869565, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"distinctValueFrequencies\": [{\"value\": \"NullValue\", \"frequency\": 1}, {\"value\": \"apple\", \"frequency\": 7}, {\"value\": \"chicken\", \"frequency\": 7}, {\"value\": \"cookie\", \"frequency\": 6}, {\"value\": \"hamburger\", \"frequency\": 7}, {\"value\": \"lasagna\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 7}, {\"value\": \"pasta\", \"frequency\": 7}, {\"value\": \"spinach\", \"frequency\": 7}, {\"value\": \"sushi\", \"frequency\": 7}, {\"value\": \"water\", \"frequency\": 7}], \"sampleValues\": [\"apple\", \"apple\", \"apple\", \"chicken\", \"cookie\", \"cookie\", \"cookie\", \"lasagna\", \"lasagna\", \"orange\", \"orange\", \"pasta\", \"pasta\", \"pasta\", \"pasta\", \"spinach\", \"spinach\", \"spinach\", \"water\", \"water\"]}, {\"fieldPath\": \"weight\", \"uniqueCount\": 10, \"uniqueProportion\": 0.14285714285714285, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10\", \"frequency\": 7}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"23\", \"frequency\": 7}, {\"value\": \"32\", \"frequency\": 7}, {\"value\": \"36\", \"frequency\": 7}, {\"value\": \"43\", \"frequency\": 7}, {\"value\": \"49\", \"frequency\": 7}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"53\", \"frequency\": 7}, {\"value\": \"72\", \"frequency\": 7}], \"sampleValues\": [\"10\", \"10\", \"10\", \"23\", \"23\", \"23\", \"32\", \"32\", \"36\", \"43\", \"43\", \"49\", \"49\", \"50\", \"50\", \"50\", \"72\", \"72\", \"72\", \"72\"]}, {\"fieldPath\": \"height\", \"uniqueCount\": 4, \"uniqueProportion\": 0.05714285714285714, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 23}, {\"value\": \"7\", \"frequency\": 8}], \"sampleValues\": [\"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"6\", \"6\", \"6\", \"7\", \"7\"]}, {\"fieldPath\": \"color\", \"uniqueCount\": 7, \"uniqueProportion\": 0.1, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"blue\", \"frequency\": 7}, {\"value\": \"brown\", \"frequency\": 14}, {\"value\": \"green\", \"frequency\": 7}, {\"value\": \"orange\", \"frequency\": 14}, {\"value\": \"red\", \"frequency\": 14}, {\"value\": \"white\", \"frequency\": 7}, {\"value\": \"yellow\", \"frequency\": 7}], \"sampleValues\": [\"blue\", \"blue\", \"brown\", \"brown\", \"brown\", \"green\", \"green\", \"green\", \"orange\", \"orange\", \"red\", \"red\", \"red\", \"red\", \"red\", \"white\", \"yellow\", \"yellow\", \"yellow\", \"yellow\"]}, {\"fieldPath\": \"healthy\", \"uniqueCount\": 2, \"uniqueProportion\": 0.028985507246376812, \"nullCount\": 1, \"nullProportion\": 0.014285714285714285, \"sampleValues\": [\"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"False\", \"None\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\", \"True\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "small.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "small.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "2nd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3rd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "4th chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Progression Quality", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 5, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 3, \"uniqueProportion\": 0.6, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"4\", \"frequency\": 1}, {\"value\": \"6\", \"frequency\": 1}], \"sampleValues\": [\"1\", \"1\", \"1\", \"4\", \"6\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 3, \"uniqueProportion\": 0.6, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 2}, {\"value\": \"6\", \"frequency\": 2}], \"sampleValues\": [\"1\", \"4\", \"4\", \"6\", \"6\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 3, \"uniqueProportion\": 0.6, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"2\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 1}], \"sampleValues\": [\"1\", \"1\", \"1\", \"2\", \"4\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 2, \"uniqueProportion\": 0.4, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 1}, {\"value\": \"5\", \"frequency\": 4}], \"sampleValues\": [\"1\", \"5\", \"5\", \"5\", \"5\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 4, \"uniqueProportion\": 0.8, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}], \"sampleValues\": [\"Alternative\", \"Catchy\", \"Didgeridoo\", \"Dreadful\", \"Dreadful\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "wa_fn_usec_hr_employee_attrition_csv.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "wa_fn_usec_hr_employee_attrition_csv.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "age", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "attrition", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "businesstravel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "dailyrate", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "department", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "distancefromhome", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "education", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "educationfield", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "employeecount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "environmentsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "gender", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobinvolvement", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "joblevel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobrole", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "maritalstatus", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "numcompaniesworked", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "over18", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "overtime", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "percentsalaryhike", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "performancerating", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "relationshipsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "standardhours", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "stockoptionlevel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "totalworkingyears", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "trainingtimeslastyear", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "worklifebalance", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearsatcompany", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearsincurrentrole", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearssincelastpromotion", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearswithcurrmanager", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:42f9e89a8684547e92b91ba826cb751a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 249, \"columnCount\": 31, \"fieldProfiles\": [{\"fieldPath\": \"age\", \"uniqueCount\": 39, \"uniqueProportion\": 0.1566265060240964, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"19\", \"frequency\": 4}, {\"value\": \"20\", \"frequency\": 1}, {\"value\": \"21\", \"frequency\": 1}, {\"value\": \"22\", \"frequency\": 5}, {\"value\": \"23\", \"frequency\": 1}, {\"value\": \"24\", \"frequency\": 4}, {\"value\": \"25\", \"frequency\": 3}, {\"value\": \"26\", \"frequency\": 4}, {\"value\": \"27\", \"frequency\": 9}, {\"value\": \"28\", \"frequency\": 4}, {\"value\": \"29\", \"frequency\": 8}, {\"value\": \"30\", \"frequency\": 16}, {\"value\": \"31\", \"frequency\": 9}, {\"value\": \"32\", \"frequency\": 15}, {\"value\": \"33\", \"frequency\": 11}, {\"value\": \"34\", \"frequency\": 14}, {\"value\": \"35\", \"frequency\": 14}, {\"value\": \"36\", \"frequency\": 13}, {\"value\": \"37\", \"frequency\": 11}, {\"value\": \"38\", \"frequency\": 11}, {\"value\": \"39\", \"frequency\": 3}, {\"value\": \"40\", \"frequency\": 7}, {\"value\": \"41\", \"frequency\": 9}, {\"value\": \"42\", \"frequency\": 3}, {\"value\": \"43\", \"frequency\": 5}, {\"value\": \"44\", \"frequency\": 4}, {\"value\": \"45\", \"frequency\": 8}, {\"value\": \"46\", \"frequency\": 7}, {\"value\": \"48\", \"frequency\": 1}, {\"value\": \"49\", \"frequency\": 3}, {\"value\": \"50\", \"frequency\": 7}, {\"value\": \"51\", \"frequency\": 8}, {\"value\": \"52\", \"frequency\": 3}, {\"value\": \"53\", \"frequency\": 4}, {\"value\": \"54\", \"frequency\": 3}, {\"value\": \"55\", \"frequency\": 3}, {\"value\": \"56\", \"frequency\": 3}, {\"value\": \"57\", \"frequency\": 1}, {\"value\": \"58\", \"frequency\": 3}, {\"value\": \"59\", \"frequency\": 6}], \"sampleValues\": [\"22\", \"24\", \"27\", \"28\", \"32\", \"32\", \"34\", \"37\", \"38\", \"40\", \"40\", \"41\", \"42\", \"44\", \"46\", \"50\", \"50\", \"51\", \"52\", \"58\"]}, {\"fieldPath\": \"attrition\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"No\", \"frequency\": 209}, {\"value\": \"Yes\", \"frequency\": 40}], \"sampleValues\": [\"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"Yes\", \"Yes\", \"Yes\"]}, {\"fieldPath\": \"businesstravel\", \"uniqueCount\": 3, \"uniqueProportion\": 0.012048192771084338, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Non-Travel\", \"frequency\": 25}, {\"value\": \"Travel_Frequently\", \"frequency\": 46}, {\"value\": \"Travel_Rarely\", \"frequency\": 178}], \"sampleValues\": [\"Non-Travel\", \"Non-Travel\", \"Non-Travel\", \"Travel_Frequently\", \"Travel_Frequently\", \"Travel_Frequently\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\", \"Travel_Rarely\"]}, {\"fieldPath\": \"dailyrate\", \"uniqueCount\": 7, \"uniqueProportion\": 0.028112449799196786, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 49}, {\"value\": \"19\", \"frequency\": 17}, {\"value\": \"2\", \"frequency\": 22}, {\"value\": \"24\", \"frequency\": 11}, {\"value\": \"32\", \"frequency\": 15}, {\"value\": \"4\", \"frequency\": 15}, {\"value\": \"514\", \"frequency\": 24}, {\"value\": \"7\", \"frequency\": 96}], \"sampleValues\": [\"0\", \"0\", \"0\", \"0\", \"0\", \"19\", \"2\", \"24\", \"32\", \"32\", \"4\", \"7\", \"7\", \"7\", \"7\", \"7\", \"7\", \"7\", \"7\", \"7\"]}, {\"fieldPath\": \"department\", \"uniqueCount\": 3, \"uniqueProportion\": 0.012048192771084338, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Human Resources\", \"frequency\": 7}, {\"value\": \"Research & Development\", \"frequency\": 175}, {\"value\": \"Sales\", \"frequency\": 67}], \"sampleValues\": [\"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Research & Development\", \"Sales\", \"Sales\", \"Sales\", \"Sales\", \"Sales\", \"Sales\"]}, {\"fieldPath\": \"distancefromhome\", \"uniqueCount\": 30, \"uniqueProportion\": 0.12048192771084337, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 42}, {\"value\": \"10\", \"frequency\": 4}, {\"value\": \"11\", \"frequency\": 6}, {\"value\": \"12\", \"frequency\": 3}, {\"value\": \"13\", \"frequency\": 1}, {\"value\": \"14\", \"frequency\": 5}, {\"value\": \"15\", \"frequency\": 3}, {\"value\": \"16\", \"frequency\": 4}, {\"value\": \"17\", \"frequency\": 1}, {\"value\": \"18\", \"frequency\": 6}, {\"value\": \"19\", \"frequency\": 5}, {\"value\": \"2\", \"frequency\": 29}, {\"value\": \"20\", \"frequency\": 4}, {\"value\": \"21\", \"frequency\": 5}, {\"value\": \"22\", \"frequency\": 2}, {\"value\": \"23\", \"frequency\": 8}, {\"value\": \"24\", \"frequency\": 4}, {\"value\": \"25\", \"frequency\": 3}, {\"value\": \"26\", \"frequency\": 6}, {\"value\": \"27\", \"frequency\": 3}, {\"value\": \"28\", \"frequency\": 2}, {\"value\": \"29\", \"frequency\": 4}, {\"value\": \"3\", \"frequency\": 16}, {\"value\": \"4\", \"frequency\": 12}, {\"value\": \"5\", \"frequency\": 14}, {\"value\": \"6\", \"frequency\": 18}, {\"value\": \"7\", \"frequency\": 11}, {\"value\": \"8\", \"frequency\": 11}, {\"value\": \"9\", \"frequency\": 17}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"10\", \"12\", \"12\", \"18\", \"19\", \"2\", \"2\", \"23\", \"24\", \"3\", \"4\", \"4\", \"4\", \"6\", \"6\", \"9\"]}, {\"fieldPath\": \"education\", \"uniqueCount\": 5, \"uniqueProportion\": 0.020080321285140562, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 31}, {\"value\": \"2\", \"frequency\": 53}, {\"value\": \"3\", \"frequency\": 95}, {\"value\": \"4\", \"frequency\": 63}, {\"value\": \"5\", \"frequency\": 7}], \"sampleValues\": [\"1\", \"1\", \"1\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\", \"4\", \"4\", \"4\"]}, {\"fieldPath\": \"educationfield\", \"uniqueCount\": 6, \"uniqueProportion\": 0.024096385542168676, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Human Resources\", \"frequency\": 4}, {\"value\": \"Life Sciences\", \"frequency\": 102}, {\"value\": \"Marketing\", \"frequency\": 28}, {\"value\": \"Medical\", \"frequency\": 85}, {\"value\": \"Other\", \"frequency\": 11}, {\"value\": \"Technical Degree\", \"frequency\": 19}], \"sampleValues\": [\"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Life Sciences\", \"Marketing\", \"Marketing\", \"Medical\", \"Medical\", \"Medical\", \"Medical\", \"Medical\", \"Medical\", \"Medical\", \"Technical Degree\", \"Technical Degree\", \"Technical Degree\"]}, {\"fieldPath\": \"employeecount\", \"uniqueCount\": 1, \"uniqueProportion\": 0.004016064257028112, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 249}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\"]}, {\"fieldPath\": \"environmentsatisfaction\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 41}, {\"value\": \"2\", \"frequency\": 56}, {\"value\": \"3\", \"frequency\": 87}, {\"value\": \"4\", \"frequency\": 65}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\", \"4\"]}, {\"fieldPath\": \"gender\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Female\", \"frequency\": 101}, {\"value\": \"Male\", \"frequency\": 148}], \"sampleValues\": [\"Female\", \"Female\", \"Female\", \"Female\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\", \"Male\"]}, {\"fieldPath\": \"jobinvolvement\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 13}, {\"value\": \"2\", \"frequency\": 66}, {\"value\": \"3\", \"frequency\": 151}, {\"value\": \"4\", \"frequency\": 19}], \"sampleValues\": [\"2\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\"]}, {\"fieldPath\": \"joblevel\", \"uniqueCount\": 5, \"uniqueProportion\": 0.020080321285140562, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 101}, {\"value\": \"2\", \"frequency\": 78}, {\"value\": \"3\", \"frequency\": 45}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 15}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\"]}, {\"fieldPath\": \"jobrole\", \"uniqueCount\": 9, \"uniqueProportion\": 0.03614457831325301, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Healthcare Representative\", \"frequency\": 16}, {\"value\": \"Human Resources\", \"frequency\": 5}, {\"value\": \"Laboratory Technician\", \"frequency\": 51}, {\"value\": \"Manager\", \"frequency\": 19}, {\"value\": \"Manufacturing Director\", \"frequency\": 25}, {\"value\": \"Research Director\", \"frequency\": 13}, {\"value\": \"Research Scientist\", \"frequency\": 60}, {\"value\": \"Sales Executive\", \"frequency\": 48}, {\"value\": \"Sales Representative\", \"frequency\": 12}], \"sampleValues\": [\"Healthcare Representative\", \"Laboratory Technician\", \"Laboratory Technician\", \"Laboratory Technician\", \"Laboratory Technician\", \"Laboratory Technician\", \"Manager\", \"Manager\", \"Manufacturing Director\", \"Manufacturing Director\", \"Research Director\", \"Research Director\", \"Research Director\", \"Research Scientist\", \"Research Scientist\", \"Sales Executive\", \"Sales Executive\", \"Sales Executive\", \"Sales Executive\", \"Sales Representative\"]}, {\"fieldPath\": \"jobsatisfaction\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 46}, {\"value\": \"2\", \"frequency\": 43}, {\"value\": \"3\", \"frequency\": 74}, {\"value\": \"4\", \"frequency\": 86}], \"sampleValues\": [\"1\", \"1\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\"]}, {\"fieldPath\": \"maritalstatus\", \"uniqueCount\": 3, \"uniqueProportion\": 0.012048192771084338, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Divorced\", \"frequency\": 57}, {\"value\": \"Married\", \"frequency\": 105}, {\"value\": \"Single\", \"frequency\": 87}], \"sampleValues\": [\"Divorced\", \"Divorced\", \"Divorced\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Married\", \"Single\", \"Single\", \"Single\", \"Single\", \"Single\", \"Single\"]}, {\"fieldPath\": \"numcompaniesworked\", \"uniqueCount\": 10, \"uniqueProportion\": 0.040160642570281124, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 33}, {\"value\": \"1\", \"frequency\": 89}, {\"value\": \"2\", \"frequency\": 25}, {\"value\": \"3\", \"frequency\": 23}, {\"value\": \"4\", \"frequency\": 24}, {\"value\": \"5\", \"frequency\": 15}, {\"value\": \"6\", \"frequency\": 9}, {\"value\": \"7\", \"frequency\": 13}, {\"value\": \"8\", \"frequency\": 8}, {\"value\": \"9\", \"frequency\": 10}], \"sampleValues\": [\"0\", \"0\", \"0\", \"0\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"2\", \"2\", \"3\", \"4\", \"4\", \"5\", \"6\", \"7\", \"7\", \"8\"]}, {\"fieldPath\": \"over18\", \"uniqueCount\": 1, \"uniqueProportion\": 0.004016064257028112, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Y\", \"frequency\": 249}], \"sampleValues\": [\"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\", \"Y\"]}, {\"fieldPath\": \"overtime\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"No\", \"frequency\": 178}, {\"value\": \"Yes\", \"frequency\": 71}], \"sampleValues\": [\"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"No\", \"Yes\", \"Yes\", \"Yes\", \"Yes\", \"Yes\", \"Yes\"]}, {\"fieldPath\": \"percentsalaryhike\", \"uniqueCount\": 15, \"uniqueProportion\": 0.060240963855421686, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"11\", \"frequency\": 32}, {\"value\": \"12\", \"frequency\": 32}, {\"value\": \"13\", \"frequency\": 34}, {\"value\": \"14\", \"frequency\": 35}, {\"value\": \"15\", \"frequency\": 16}, {\"value\": \"16\", \"frequency\": 13}, {\"value\": \"17\", \"frequency\": 13}, {\"value\": \"18\", \"frequency\": 16}, {\"value\": \"19\", \"frequency\": 10}, {\"value\": \"20\", \"frequency\": 9}, {\"value\": \"21\", \"frequency\": 9}, {\"value\": \"22\", \"frequency\": 17}, {\"value\": \"23\", \"frequency\": 9}, {\"value\": \"24\", \"frequency\": 2}, {\"value\": \"25\", \"frequency\": 2}], \"sampleValues\": [\"11\", \"11\", \"11\", \"11\", \"11\", \"12\", \"12\", \"12\", \"12\", \"13\", \"13\", \"13\", \"14\", \"14\", \"15\", \"18\", \"18\", \"18\", \"19\", \"22\"]}, {\"fieldPath\": \"performancerating\", \"uniqueCount\": 2, \"uniqueProportion\": 0.008032128514056224, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"3\", \"frequency\": 201}, {\"value\": \"4\", \"frequency\": 48}], \"sampleValues\": [\"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\"]}, {\"fieldPath\": \"relationshipsatisfaction\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 47}, {\"value\": \"2\", \"frequency\": 32}, {\"value\": \"3\", \"frequency\": 81}, {\"value\": \"4\", \"frequency\": 89}], \"sampleValues\": [\"1\", \"1\", \"1\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\"]}, {\"fieldPath\": \"standardhours\", \"uniqueCount\": 1, \"uniqueProportion\": 0.004016064257028112, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"80\", \"frequency\": 249}], \"sampleValues\": [\"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\", \"80\"]}, {\"fieldPath\": \"stockoptionlevel\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 117}, {\"value\": \"1\", \"frequency\": 98}, {\"value\": \"2\", \"frequency\": 20}, {\"value\": \"3\", \"frequency\": 14}], \"sampleValues\": [\"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"0\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\"]}, {\"fieldPath\": \"totalworkingyears\", \"uniqueCount\": 38, \"uniqueProportion\": 0.15261044176706828, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 2}, {\"value\": \"1\", \"frequency\": 14}, {\"value\": \"10\", \"frequency\": 27}, {\"value\": \"11\", \"frequency\": 6}, {\"value\": \"12\", \"frequency\": 13}, {\"value\": \"13\", \"frequency\": 7}, {\"value\": \"14\", \"frequency\": 4}, {\"value\": \"15\", \"frequency\": 3}, {\"value\": \"16\", \"frequency\": 11}, {\"value\": \"17\", \"frequency\": 12}, {\"value\": \"18\", \"frequency\": 2}, {\"value\": \"19\", \"frequency\": 3}, {\"value\": \"2\", \"frequency\": 7}, {\"value\": \"20\", \"frequency\": 3}, {\"value\": \"21\", \"frequency\": 6}, {\"value\": \"22\", \"frequency\": 5}, {\"value\": \"23\", \"frequency\": 6}, {\"value\": \"24\", \"frequency\": 3}, {\"value\": \"25\", \"frequency\": 3}, {\"value\": \"26\", \"frequency\": 1}, {\"value\": \"28\", \"frequency\": 2}, {\"value\": \"29\", \"frequency\": 1}, {\"value\": \"3\", \"frequency\": 4}, {\"value\": \"30\", \"frequency\": 1}, {\"value\": \"31\", \"frequency\": 1}, {\"value\": \"32\", \"frequency\": 1}, {\"value\": \"33\", \"frequency\": 1}, {\"value\": \"34\", \"frequency\": 1}, {\"value\": \"36\", \"frequency\": 1}, {\"value\": \"37\", \"frequency\": 1}, {\"value\": \"38\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"40\", \"frequency\": 1}, {\"value\": \"5\", \"frequency\": 14}, {\"value\": \"6\", \"frequency\": 21}, {\"value\": \"7\", \"frequency\": 19}, {\"value\": \"8\", \"frequency\": 15}, {\"value\": \"9\", \"frequency\": 17}], \"sampleValues\": [\"15\", \"15\", \"16\", \"17\", \"17\", \"21\", \"22\", \"23\", \"23\", \"28\", \"3\", \"33\", \"38\", \"6\", \"6\", \"6\", \"6\", \"7\", \"8\", \"9\"]}, {\"fieldPath\": \"trainingtimeslastyear\", \"uniqueCount\": 7, \"uniqueProportion\": 0.028112449799196786, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 8}, {\"value\": \"1\", \"frequency\": 18}, {\"value\": \"2\", \"frequency\": 90}, {\"value\": \"3\", \"frequency\": 87}, {\"value\": \"4\", \"frequency\": 13}, {\"value\": \"5\", \"frequency\": 24}, {\"value\": \"6\", \"frequency\": 9}], \"sampleValues\": [\"0\", \"1\", \"1\", \"1\", \"2\", \"2\", \"2\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"4\", \"5\", \"5\", \"5\", \"6\"]}, {\"fieldPath\": \"worklifebalance\", \"uniqueCount\": 4, \"uniqueProportion\": 0.01606425702811245, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 14}, {\"value\": \"2\", \"frequency\": 64}, {\"value\": \"3\", \"frequency\": 146}, {\"value\": \"4\", \"frequency\": 25}], \"sampleValues\": [\"1\", \"2\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"3\", \"4\", \"4\", \"4\", \"4\"]}, {\"fieldPath\": \"yearsatcompany\", \"uniqueCount\": 29, \"uniqueProportion\": 0.11646586345381527, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 7}, {\"value\": \"1\", \"frequency\": 33}, {\"value\": \"10\", \"frequency\": 18}, {\"value\": \"11\", \"frequency\": 9}, {\"value\": \"12\", \"frequency\": 4}, {\"value\": \"13\", \"frequency\": 3}, {\"value\": \"14\", \"frequency\": 2}, {\"value\": \"15\", \"frequency\": 4}, {\"value\": \"16\", \"frequency\": 2}, {\"value\": \"17\", \"frequency\": 3}, {\"value\": \"19\", \"frequency\": 1}, {\"value\": \"2\", \"frequency\": 22}, {\"value\": \"20\", \"frequency\": 4}, {\"value\": \"21\", \"frequency\": 1}, {\"value\": \"22\", \"frequency\": 4}, {\"value\": \"24\", \"frequency\": 3}, {\"value\": \"25\", \"frequency\": 2}, {\"value\": \"27\", \"frequency\": 1}, {\"value\": \"3\", \"frequency\": 20}, {\"value\": \"33\", \"frequency\": 2}, {\"value\": \"37\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 19}, {\"value\": \"40\", \"frequency\": 1}, {\"value\": \"5\", \"frequency\": 31}, {\"value\": \"6\", \"frequency\": 11}, {\"value\": \"7\", \"frequency\": 15}, {\"value\": \"8\", \"frequency\": 9}, {\"value\": \"9\", \"frequency\": 17}], \"sampleValues\": [\"10\", \"12\", \"16\", \"2\", \"2\", \"2\", \"20\", \"22\", \"22\", \"3\", \"33\", \"37\", \"4\", \"4\", \"4\", \"5\", \"7\", \"7\", \"8\", \"9\"]}, {\"fieldPath\": \"yearsincurrentrole\", \"uniqueCount\": 19, \"uniqueProportion\": 0.07630522088353414, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 43}, {\"value\": \"1\", \"frequency\": 8}, {\"value\": \"10\", \"frequency\": 6}, {\"value\": \"11\", \"frequency\": 5}, {\"value\": \"12\", \"frequency\": 2}, {\"value\": \"13\", \"frequency\": 5}, {\"value\": \"14\", \"frequency\": 2}, {\"value\": \"15\", \"frequency\": 2}, {\"value\": \"16\", \"frequency\": 1}, {\"value\": \"17\", \"frequency\": 1}, {\"value\": \"18\", \"frequency\": 2}, {\"value\": \"2\", \"frequency\": 61}, {\"value\": \"3\", \"frequency\": 23}, {\"value\": \"4\", \"frequency\": 19}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 4}, {\"value\": \"7\", \"frequency\": 32}, {\"value\": \"8\", \"frequency\": 19}, {\"value\": \"9\", \"frequency\": 9}], \"sampleValues\": [\"1\", \"10\", \"10\", \"11\", \"15\", \"17\", \"18\", \"2\", \"2\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"3\", \"5\", \"7\", \"8\", \"9\"]}, {\"fieldPath\": \"yearssincelastpromotion\", \"uniqueCount\": 15, \"uniqueProportion\": 0.060240963855421686, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 96}, {\"value\": \"1\", \"frequency\": 56}, {\"value\": \"10\", \"frequency\": 1}, {\"value\": \"11\", \"frequency\": 4}, {\"value\": \"12\", \"frequency\": 3}, {\"value\": \"13\", \"frequency\": 1}, {\"value\": \"15\", \"frequency\": 5}, {\"value\": \"2\", \"frequency\": 35}, {\"value\": \"3\", \"frequency\": 8}, {\"value\": \"4\", \"frequency\": 14}, {\"value\": \"5\", \"frequency\": 10}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 7}, {\"value\": \"8\", \"frequency\": 3}, {\"value\": \"9\", \"frequency\": 4}], \"sampleValues\": [\"0\", \"0\", \"0\", \"1\", \"1\", \"1\", \"1\", \"1\", \"11\", \"15\", \"15\", \"15\", \"2\", \"2\", \"2\", \"2\", \"3\", \"5\", \"7\", \"9\"]}, {\"fieldPath\": \"yearswithcurrmanager\", \"uniqueCount\": 16, \"uniqueProportion\": 0.0642570281124498, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"0\", \"frequency\": 44}, {\"value\": \"1\", \"frequency\": 15}, {\"value\": \"10\", \"frequency\": 2}, {\"value\": \"11\", \"frequency\": 4}, {\"value\": \"12\", \"frequency\": 3}, {\"value\": \"13\", \"frequency\": 2}, {\"value\": \"15\", \"frequency\": 4}, {\"value\": \"17\", \"frequency\": 1}, {\"value\": \"2\", \"frequency\": 49}, {\"value\": \"3\", \"frequency\": 25}, {\"value\": \"4\", \"frequency\": 15}, {\"value\": \"5\", \"frequency\": 6}, {\"value\": \"6\", \"frequency\": 6}, {\"value\": \"7\", \"frequency\": 35}, {\"value\": \"8\", \"frequency\": 25}, {\"value\": \"9\", \"frequency\": 13}], \"sampleValues\": [\"1\", \"1\", \"11\", \"12\", \"13\", \"15\", \"15\", \"2\", \"2\", \"2\", \"2\", \"3\", \"3\", \"5\", \"7\", \"7\", \"7\", \"8\", \"8\", \"8\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json new file mode 100644 index 00000000000000..4244681c49cc75 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/local/golden_mces_single_file.json @@ -0,0 +1,774 @@ +[ +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "chord_progressions_avro.avro", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_avro.avro", + "platform": "urn:li:dataPlatform:file", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=Root].[type=double].Progression Quality", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].1st chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].2nd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].3rd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=string].4th chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests\"}, \"name\": \"tests\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec641f2e55b5b507547420e7fe726e43", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration\"}, \"name\": \"integration\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c447df2838a56ea10746b2b29c6e1f55", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec641f2e55b5b507547420e7fe726e43\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration/s3\"}, \"name\": \"s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:7a7bb5d2c697f75af94470ff692c62fc", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:c447df2838a56ea10746b2b29c6e1f55\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration/s3/test_data\"}, \"name\": \"test_data\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0cfba651f28915437b9ca20b860794b8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:7a7bb5d2c697f75af94470ff692c62fc\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system\"}, \"name\": \"local_system\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:68e98abbe42abc5b7d18ecc59e99394b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:0cfba651f28915437b9ca20b860794b8\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:68e98abbe42abc5b7d18ecc59e99394b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0e95508357b8eda0da27f14ac7ae620b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:f1af6f7ad766fd11aaa792faa5670cfd\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"file\", \"instance\": \"PROD\", \"folder_abs_path\": \"tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:file\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:0b11f7e4b6df7e17a294482039b93289", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:0e95508357b8eda0da27f14ac7ae620b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:0b11f7e4b6df7e17a294482039b93289\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:file,tests/integration/s3/test_data/local_system/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "value": "{\"timestampMillis\": 1615443388097, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 28, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"1st chord\", \"uniqueCount\": 5, \"uniqueProportion\": 0.17857142857142858, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 5}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 10}, {\"value\": \"5\", \"frequency\": 7}, {\"value\": \"6\", \"frequency\": 5}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\"]}, {\"fieldPath\": \"2nd chord\", \"uniqueCount\": 7, \"uniqueProportion\": 0.25, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 10}, {\"value\": \"2\", \"frequency\": 2}, {\"value\": \"3\", \"frequency\": 2}, {\"value\": \"4\", \"frequency\": 6}, {\"value\": \"5\", \"frequency\": 5}, {\"value\": \"6\", \"frequency\": 2}, {\"value\": \"7\", \"frequency\": 1}], \"sampleValues\": [\"1\", \"1\", \"1\", \"1\", \"1\", \"1\", \"2\", \"2\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"6\"]}, {\"fieldPath\": \"3rd chord\", \"uniqueCount\": 6, \"uniqueProportion\": 0.21428571428571427, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"1\", \"frequency\": 3}, {\"value\": \"3\", \"frequency\": 1}, {\"value\": \"4\", \"frequency\": 9}, {\"value\": \"5\", \"frequency\": 8}, {\"value\": \"6\", \"frequency\": 5}, {\"value\": \"7\", \"frequency\": 2}], \"sampleValues\": [\"1\", \"1\", \"1\", \"3\", \"4\", \"4\", \"4\", \"4\", \"4\", \"5\", \"5\", \"5\", \"5\", \"5\", \"5\", \"5\", \"6\", \"6\", \"6\", \"7\"]}, {\"fieldPath\": \"4th chord\", \"uniqueCount\": 20, \"uniqueProportion\": 0.7142857142857143, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Alternative\", \"frequency\": 1}, {\"value\": \"Catchy\", \"frequency\": 1}, {\"value\": \"Didgeridoo\", \"frequency\": 1}, {\"value\": \"Dreadful\", \"frequency\": 2}, {\"value\": \"Endless\", \"frequency\": 1}, {\"value\": \"Energetic\", \"frequency\": 1}, {\"value\": \"Flamenco\", \"frequency\": 2}, {\"value\": \"Folk\", \"frequency\": 2}, {\"value\": \"Grunge\", \"frequency\": 1}, {\"value\": \"Jazz\", \"frequency\": 1}, {\"value\": \"Love\", \"frequency\": 1}, {\"value\": \"Memories\", \"frequency\": 1}, {\"value\": \"Moody\", \"frequency\": 2}, {\"value\": \"Pop\", \"frequency\": 2}, {\"value\": \"Rebellious\", \"frequency\": 1}, {\"value\": \"Sad\", \"frequency\": 3}, {\"value\": \"Simple\", \"frequency\": 2}, {\"value\": \"Sweet\", \"frequency\": 1}, {\"value\": \"Wildside\", \"frequency\": 1}, {\"value\": \"Wistful\", \"frequency\": 1}], \"sampleValues\": [\"Alternative\", \"Didgeridoo\", \"Dreadful\", \"Dreadful\", \"Endless\", \"Energetic\", \"Flamenco\", \"Flamenco\", \"Folk\", \"Grunge\", \"Memories\", \"Moody\", \"Rebellious\", \"Sad\", \"Sad\", \"Sad\", \"Simple\", \"Simple\", \"Sweet\", \"Wistful\"]}, {\"fieldPath\": \"Progression Quality\", \"uniqueCount\": 1, \"uniqueProportion\": 0.03571428571428571, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"NaN\", \"frequency\": 28}], \"sampleValues\": [\"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\", \"nan\"]}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json new file mode 100644 index 00000000000000..8dc0de2ee4fe04 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition.json @@ -0,0 +1,615 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:050fedde7a12cb8c8447db8d298f5577\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:86297df39321e4948dbe8b8e941de98b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:273fbeff7bd9ecb74982205aadd77994\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_csv", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json new file mode 100644 index 00000000000000..3b5a24dda766fa --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_exclude.json @@ -0,0 +1,451 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:050fedde7a12cb8c8447db8d298f5577\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:86297df39321e4948dbe8b8e941de98b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:273fbeff7bd9ecb74982205aadd77994\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_exclude.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json new file mode 100644 index 00000000000000..f8d42c8f824cef --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_filename.json @@ -0,0 +1,451 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:050fedde7a12cb8c8447db8d298f5577\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:86297df39321e4948dbe8b8e941de98b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:273fbeff7bd9ecb74982205aadd77994\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "folder_aaa.food_csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "folder_aaa.food_csv", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_csv,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_filename.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json new file mode 100644 index 00000000000000..b2e4167219b2bb --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_no_partition_glob.json @@ -0,0 +1,451 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:050fedde7a12cb8c8447db8d298f5577\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:86297df39321e4948dbe8b8e941de98b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:273fbeff7bd9ecb74982205aadd77994\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_no_partition_glob.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json new file mode 100644 index 00000000000000..74961ccb92269c --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_basic.json @@ -0,0 +1,1058 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:050fedde7a12cb8c8447db8d298f5577\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:86297df39321e4948dbe8b8e941de98b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:273fbeff7bd9ecb74982205aadd77994\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "folder_aaa.pokemon_abilities_json", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.short_effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "is_main_series", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.is_hidden", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.slot", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_basic.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json new file mode 100644 index 00000000000000..f51f6784e5d845 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_folder_partition_keyval.json @@ -0,0 +1,1058 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:050fedde7a12cb8c8447db8d298f5577", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:86297df39321e4948dbe8b8e941de98b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:050fedde7a12cb8c8447db8d298f5577\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:273fbeff7bd9ecb74982205aadd77994", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:86297df39321e4948dbe8b8e941de98b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"UAT\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:273fbeff7bd9ecb74982205aadd77994\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "folder_aaa.pokemon_abilities_json", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "folder_aaa.pokemon_abilities_json", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "effect_changes", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_changes.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "effect_entries.short_effect", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.flavor_text", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "flavor_text_entries.version_group.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "generation.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "id", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "is_main_series", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.language.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "names.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.is_hidden", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "dict", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.pokemon.url", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pokemon.slot", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/pokemon_abilities_json,UAT)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:ec362903c4c7de60197fcc7b7a79e4c2\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "folder_partition_keyval.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json new file mode 100644 index 00000000000000..b00d0c352bec92 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_multiple_files.json @@ -0,0 +1,2287 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"test-platform-instance\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:647eefb4dfda8695baf1aa0775d78689", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:69d5498bfa21a9ba29b81cb5b273f159", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:69d5498bfa21a9ba29b81cb5b273f159", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:69d5498bfa21a9ba29b81cb5b273f159", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:69d5498bfa21a9ba29b81cb5b273f159", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:647eefb4dfda8695baf1aa0775d78689\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dd2ab8c07175d16dae2a7031cb6fa4af", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dd2ab8c07175d16dae2a7031cb6fa4af", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dd2ab8c07175d16dae2a7031cb6fa4af", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:dd2ab8c07175d16dae2a7031cb6fa4af", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:69d5498bfa21a9ba29b81cb5b273f159\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"test-platform\", \"instance\": \"test-platform-instance\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:test-platform\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:dd2ab8c07175d16dae2a7031cb6fa4af\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "2", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Br \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Ca \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Cl \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Cond (\u00b5S/cm)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "DO (mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "DOC [mg/L C]", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "F \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "K \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Lat (\u00b0N)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Long (\u00b0W)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Mg \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "NH3-N \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "NO3-N+NO2-N \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Na \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "PO4-P \n(mg P/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Park ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "SO4-S \n(mg/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "SUVA, 254nm", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Sampling Date", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Secchi Depth (m)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Site ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "TDN \n(mg N/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "TDP \n(mg P/L)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "UV Absorbance, 254nm", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Water Temp (\u00b0C)", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "d18O", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "dD", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "field29", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "pH", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/NPS.7.1.package_data_NPS.6.1_ARCN_Lakes_ChemistryData_v1_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "chord_progressions_avro.avro", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_avro.avro", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=Root].[type=double].Progression Quality", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].1st chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].2nd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].3rd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=string].4th chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "chord_progressions_csv.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_csv.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "2nd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3rd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "4th chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Progression Quality", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "countries_json.json", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "countries_json.json", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "countries", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "list", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "countries.code", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "countries.name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "str", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/countries_json.json,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "food_parquet.parquet", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "food_parquet.parquet", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "color", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "healthy", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "bool", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "height", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "name", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "weight", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int64", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/food_parquet.parquet,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "small.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "small.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "1st chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "2nd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "3rd chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "4th chord", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Progression Quality", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/small.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "wa_fn_usec_hr_employee_attrition_csv.csv", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "wa_fn_usec_hr_employee_attrition_csv.csv", + "platform": "urn:li:dataPlatform:test-platform", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "age", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "attrition", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "businesstravel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "dailyrate", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "department", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "distancefromhome", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "education", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "educationfield", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "employeecount", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "environmentsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "gender", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobinvolvement", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "joblevel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobrole", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "jobsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "maritalstatus", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "numcompaniesworked", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "over18", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "overtime", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "percentsalaryhike", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "performancerating", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "relationshipsatisfaction", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "standardhours", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "stockoptionlevel", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "totalworkingyears", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "trainingtimeslastyear", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "worklifebalance", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearsatcompany", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearsincurrentrole", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearssincelastpromotion", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "yearswithcurrmanager", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:test-platform,test-platform-instance.my-test-bucket/folder_a/folder_aa/folder_aaa/wa_fn_usec_hr_employee_attrition_csv.csv,DEV)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5d452ddd1a2aabe2c4f334e0f229a13b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "multiple_files.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json new file mode 100644 index 00000000000000..cfff856ff8ef7a --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/golden-files/s3/golden_mces_single_file.json @@ -0,0 +1,451 @@ +[ +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"PROD\", \"bucket_name\": \"my-test-bucket\"}, \"name\": \"my-test-bucket\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2151647ff17bde0f948909d19fa91b9b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"S3 bucket\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"PROD\", \"folder_abs_path\": \"my-test-bucket/folder_a\"}, \"name\": \"folder_a\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:2151647ff17bde0f948909d19fa91b9b\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"PROD\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa\"}, \"name\": \"folder_aa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:a8aa32e8169b2ecc7ab4f3389c79124c\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "value": "{\"customProperties\": {\"platform\": \"s3\", \"instance\": \"PROD\", \"folder_abs_path\": \"my-test-bucket/folder_a/folder_aa/folder_aaa\"}, \"name\": \"folder_aaa\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:s3\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "value": "{\"typeNames\": [\"Folder\"]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:4f62b9a3e6794ee2cd4160bc0bbd8e15\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": null, + "name": "chord_progressions_avro.avro", + "qualifiedName": null, + "description": "", + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "chord_progressions_avro.avro", + "platform": "urn:li:dataPlatform:s3", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=Root].[type=double].Progression Quality", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].1st chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].2nd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=long].3rd chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "[version=2.0].[type=Root].[type=string].4th chord", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +}, +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "value": "{\"container\": \"urn:li:container:5abb7acbb8783b9e2d266c15bf7cebc0\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_file.json", + "registryName": null, + "registryVersion": null, + "properties": null + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json new file mode 100644 index 00000000000000..472fea07f44425 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition.json @@ -0,0 +1,17 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/{table}/*.*" + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_exclude.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_exclude.json new file mode 100644 index 00000000000000..2736e81ece1010 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_exclude.json @@ -0,0 +1,20 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/{table}/*.*", + "exclude": [ + "**/food_csv/**" + ] + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_filename.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_filename.json new file mode 100644 index 00000000000000..c7efcf323f36a6 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_filename.json @@ -0,0 +1,19 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/*.*", + "file_types": ["csv"], + "table_name": "{dept}.{table}" + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_glob.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_glob.json new file mode 100644 index 00000000000000..ad548271c25905 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_no_partition_glob.json @@ -0,0 +1,20 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_spec": { + "include": "s3://my-test-bucket/*/folder_aa/*/{table}/*.*", + "exclude": [ + "**/food_csv/**" + ] + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_basic.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_basic.json new file mode 100644 index 00000000000000..99850657e04d30 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_basic.json @@ -0,0 +1,21 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/{partition[0]}/{partition[1]}/*.*", + "table_name": "{dept}.{table}", + "exclude":[ + "**/folder_aaaa/**" + ] + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_keyval.json b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_keyval.json new file mode 100644 index 00000000000000..cab9fe3ffe518a --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/folder_partition_keyval.json @@ -0,0 +1,18 @@ +{ + "type": "s3", + "config": { + "env": "UAT", + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/{dept}/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.*", + "table_name": "{dept}.{table}" + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/multiple_files.json b/metadata-ingestion/tests/integration/s3/sources/s3/multiple_files.json new file mode 100644 index 00000000000000..2b7b08fabc9834 --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/multiple_files.json @@ -0,0 +1,19 @@ +{ + "type": "s3", + "config": { + "platform": "test-platform", + "platform_instance": "test-platform-instance", + "env": "DEV", + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/*.*" + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/sources/s3/single_file.json b/metadata-ingestion/tests/integration/s3/sources/s3/single_file.json new file mode 100644 index 00000000000000..b4f164ea189d6e --- /dev/null +++ b/metadata-ingestion/tests/integration/s3/sources/s3/single_file.json @@ -0,0 +1,16 @@ +{ + "type": "s3", + "config": { + "path_spec": { + "include": "s3://my-test-bucket/folder_a/folder_aa/folder_aaa/chord_progressions_avro.avro" + }, + "aws_config": { + "aws_region": "us-east-1", + "aws_access_key_id": "testing", + "aws_secret_access_key": "testing" + }, + "profiling": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/s3/test_data/cases_1/chord_progressions_avro.avro b/metadata-ingestion/tests/integration/s3/test_data/cases_1/chord_progressions_avro.avro new file mode 100644 index 0000000000000000000000000000000000000000..8a6d9df66bb79615abbee50ea71d92d5043134c4 GIT binary patch literal 1024 zcmb7DO>5gg5KR|jgb>0M6LJ(fhd`Sin%+vjAS5KQ4>^P!*4l}@?Rv+oR%x9OdhRWU zmi(FghTxwNdg>48p~ucDrYqTPAiC{)^WMyxT^$ZSyl8d#g~7E91!RFPm0GUOMK**H z|IxZi?PLt;yCgM`p%Lk7l4?GxkRGCS=W_{46jXD1^_P29*d!ay1T%}%)cHGwLY=2) z*Q<#ayw>7*f_3vu!5ce#qC;VgTo~%eU~Zr&Bz;d8~5}jQy-QE6N z-<^K{_VD@s*L3CU#@m~a1&pzfEx%V5jOJEe09&fQ-(O~2w8?EYoOqZN^mVtS$iYZ~ z=&6V_A?s`z;NrYgb)xluB4F{h7K+k%L7SW+_HtW;$-&CZGa)q(Sg?cYqX(mZfi7K} zb#@fJ3Z2!)49S|tTqIn5#G+kO>b$lH*OH>`@yko!(j}!i_Hl7#odX!5kxh}Ui;a|4pzPNi>`j9 z2dj<8-TUhH^?CE@?e6{N`X#%F9+J3>J8?#a7(25q?WiG-b=FPdBVqT?893>LC&pXllZSQ(-x4lI#* zeZvtwUu3Q(Jx8}hSz|6GWlfh)REsG#Y*(}f>Ke#vqA3mCf$B2lqBrz39rcbxUv*i+ z{)BDaWS=u&JjTY-<;aay%1N$L!yuTuht}K7%D>-WqE|KwZ=e3=&MLXPSyO+x_U-9J zvcFhKv+Jo+l*CfWs3n>z8%aE862D$a{40|EjLF6qm8Wik*(e(!;0L}4zJANF+9U)I zq6M5X`<@nta9K*3mXHY@dKKsTDrewSkvXXJMfg?d~ zETID9R&fR9=gb7^DaI=MK`hNiLu|2;st+`OitZ1Q?E9ujuibq? zO$wyphhCCM$|#X!%#$_M6xoY}UB4a)?%Z{LirLIPyF!`BqkeA{ORIV0v9Y9b3{uKz2v8{ry1Q+)U!uW zz(dpWGIutA^fYq*ZF5Z21cOp z5!J4Sdgs*~>8hbTnx_SRJ@Us6+RF?rUA7gKx-c}I@swdp$Exfj-Bf*15q%NFpJr4G z#C2xy95cj?R<;2rsNk)y*(O%dBk@12>$+it7TmQR&s5x4^ zt}6PrOKZx|C0BIESW?bdbm4=8+D=v1(`^f><*nRSJ_G5VSf1hKx3;%RnP()q&b8Zb zb?*c{H~F*6-@{QE>~b8p4-bwXa$J$C4!AnU4Y=BX(yG-t{s@V1Cge9qJHZ}is)h0e zjvH}s4&JDwnu9IwIgamveDzJiG=_b?Q#>rRPNhc2m#q8xTY6Pe$4yu4mQI9Ydmva% zO)?cj&_@HoENN2r0A+?s_srX?ak{+S)?{lRXxu2=pUO3%EbPdWy%Fr|mBWXqBeZ2y z9La1OD&Kj6cDmc1E}3<|{}9gdrM$$CJfYS!#cs#z*DJ$bH*cVvWOY2L270V2_yxA} zm7IE5r98pv48?Biw9g;t=nKgDa>F=M8pep~tu;@3repNB_XB>nlk0Wcn!@iQCWuF8 z*eg^{0J}x88=Zr;o+}Dge%|-s9nQD4 zX^t8>PiV~KwAQryUBeO^LF|{rXpS%Qb*Rnylf&(HNrrqved&;wTzY}UtMw+#1kq;~6L*{neHu%`QlY_%MzP^5Wk>BZ6B@ve?FIaBY2{$cZp z*sTod-8$YK-X9n9f|Y}Y)WiN1#`r$058&C6Vd743s?&Z(C)&SukD+Pa2F5OrjT4(C zZg2_)9<4$>1+nCIP`)z4R*Q^kn@XY4pdZpL_FM2Is0ri@P~Ln16I$LEXlLb7Xxge# z3bwo!+%;I{Ch~zgZamqC=b;(Hn6&!vd?UQ8z^QQ@1YiKsz`h+03D){sC3mK>DnH-14-3a1|aow@;Gaazyv>Q1x%gj&qBC*EYis*Czd5gg5KR|jgb>0M6LJ(fhd`Sin%+vjAS5KQ4>^P!*4l}@?Rv+oR%x9OdhRWU zmi(FghTxwNdg>48p~ucDrYqTPAiC{)^WMyxT^$ZSyl8d#g~7E91!RFPm0GUOMK**H z|IxZi?PLt;yCgM`p%Lk7l4?GxkRGCS=W_{46jXD1^_P29*d!ay1T%}%)cHGwLY=2) z*Q<#ayw>7*f_3vu!5ce#qC;VgTo~%eU~Zr&Bz;d8~5}jQy-QE6N z-<^K{_VD@s*L3CU#@m~a1&pzfEx%V5jOJEe09&fQ-(O~2w8?EYoOqZN^mVtS$iYZ~ z=&6V_A?s`z;NrYgb)xluB4F{h7K+k%L7SW+_HtW;$-&CZGa)q(Sg?cYqX(mZfi7K} zb#@fJ3Z2!)49S|tTqIn5#G+kO>b$lH*OH>`@yko!(j}!i_Hl7#odX!5kx7(25q?WiG-b=FPdBVqT?893>LC&pXllZSQ(-x4lI#* zeZvtwUu3Q(Jx8}hSz|6GWlfh)REsG#Y*(}f>Ke#vqA3mCf$B2lqBrz39rcbxUv*i+ z{)BDaWS=u&JjTY-<;aay%1N$L!yuTuht}K7%D>-WqE|KwZ=e3=&MLXPSyO+x_U-9J zvcFhKv+Jo+l*CfWs3n>z8%aE862D$a{40|EjLF6qm8Wik*(e(!;0L}4zJANF+9U)I zq6M5X`<@nta9K*3mXHY@dKKsTDrewSkvXXJMfg?d~ zETID9R&fR9=gb7^DaI=MK`hNiLu|2;st+`OitZ1Q?E9ujuibq? zO$wyphhCCM$|#X!%#$_M6xoY}UB4a)?%Z{LirLIPyF!`BqkeA{ORIV0v9Y9b3{uKz2v8{ry1Q+)U!uW zz(dpWGIutA^fYq*ZF5Z21cOp z5!J4Sdgs*~>8hbTnx_SRJ@Us6+RF?rUA7gKx-c}I@swdp$Exfj-Bf*15q%NFpJr4G z#C2xy95cj?R<;2rsNk)y*(O%dBk@12>$+it7TmQR&s5x4^ zt}6PrOKZx|C0BIESW?bdbm4=8+D=v1(`^f><*nRSJ_G5VSf1hKx3;%RnP()q&b8Zb zb?*c{H~F*6-@{QE>~b8p4-bwXa$J$C4!AnU4Y=BX(yG-t{s@V1Cge9qJHZ}is)h0e zjvH}s4&JDwnu9IwIgamveDzJiG=_b?Q#>rRPNhc2m#q8xTY6Pe$4yu4mQI9Ydmva% zO)?cj&_@HoENN2r0A+?s_srX?ak{+S)?{lRXxu2=pUO3%EbPdWy%Fr|mBWXqBeZ2y z9La1OD&Kj6cDmc1E}3<|{}9gdrM$$CJfYS!#cs#z*DJ$bH*cVvWOY2L270V2_yxA} zm7IE5r98pv48?Biw9g;t=nKgDa>F=M8pep~tu;@3repNB_XB>nlk0Wcn!@iQCWuF8 z*eg^{0J}x88=Zr;o+}Dge%|-s9nQD4 zX^t8>PiV~KwAQryUBeO^LF|{rXpS%Qb*Rnylf&(HNrrqved&;wTzY}UtMw+#1kq;~6L*{neHu%`QlY_%MzP^5Wk>BZ6B@ve?FIaBY2{$cZp z*sTod-8$YK-X9n9f|Y}Y)WiN1#`r$058&C6Vd743s?&Z(C)&SukD+Pa2F5OrjT4(C zZg2_)9<4$>1+nCIP`)z4R*Q^kn@XY4pdZpL_FM2Is0ri@P~Ln16I$LEXlLb7Xxge# z3bwo!+%;I{Ch~zgZamqC=b;(Hn6&!vd?UQ8z^QQ@1YiKsz`h+03D){sC3mK>DnH-14-3a1|aow@;Gaazyv>Q1x%gj&qBC*EYis*Czd7(25q?WiG-b=FPdBVqT?893>LC&pXllZSQ(-x4lI#* zeZvtwUu3Q(Jx8}hSz|6GWlfh)REsG#Y*(}f>Ke#vqA3mCf$B2lqBrz39rcbxUv*i+ z{)BDaWS=u&JjTY-<;aay%1N$L!yuTuht}K7%D>-WqE|KwZ=e3=&MLXPSyO+x_U-9J zvcFhKv+Jo+l*CfWs3n>z8%aE862D$a{40|EjLF6qm8Wik*(e(!;0L}4zJANF+9U)I zq6M5X`<@nta9K*3mXHY@dKKsTDrewSkvXXJMfg?d~ zETID9R&fR9=gb7^DaI=MK`hNiLu|2;st+`OitZ1Q?E9ujuibq? zO$wyphhCCM$|#X!%#$_M6xoY}UB4a)?%Z{LirLIPyF!`BqkeA{ORIV0v9Y9b3{uKz2v8{ry1Q+)U!uW zz(dpWGIutA^fYq*ZF5Z21cOp z5!J4Sdgs*~>8hbTnx_SRJ@Us6+RF?rUA7gKx-c}I@swdp$Exfj-Bf*15q%NFpJr4G z#C2xy95cj?R<;2rsNk)y*(O%dBk@12>$+it7TmQR&s5x4^ zt}6PrOKZx|C0BIESW?bdbm4=8+D=v1(`^f><*nRSJ_G5VSf1hKx3;%RnP()q&b8Zb zb?*c{H~F*6-@{QE>~b8p4-bwXa$J$C4!AnU4Y=BX(yG-t{s@V1Cge9qJHZ}is)h0e zjvH}s4&JDwnu9IwIgamveDzJiG=_b?Q#>rRPNhc2m#q8xTY6Pe$4yu4mQI9Ydmva% zO)?cj&_@HoENN2r0A+?s_srX?ak{+S)?{lRXxu2=pUO3%EbPdWy%Fr|mBWXqBeZ2y z9La1OD&Kj6cDmc1E}3<|{}9gdrM$$CJfYS!#cs#z*DJ$bH*cVvWOY2L270V2_yxA} zm7IE5r98pv48?Biw9g;t=nKgDa>F=M8pep~tu;@3repNB_XB>nlk0Wcn!@iQCWuF8 z*eg^{0J}x88=Zr;o+}Dge%|-s9nQD4 zX^t8>PiV~KwAQryUBeO^LF|{rXpS%Qb*Rnylf&(HNrrqved&;wTzY}UtMw+#1kq;~6L*{neHu%`QlY_%MzP^5Wk>BZ6B@ve?FIaBY2{$cZp z*sTod-8$YK-X9n9f|Y}Y)WiN1#`r$058&C6Vd743s?&Z(C)&SukD+Pa2F5OrjT4(C zZg2_)9<4$>1+nCIP`)z4R*Q^kn@XY4pdZpL_FM2Is0ri@P~Ln16I$LEXlLb7Xxge# z3bwo!+%;I{Ch~zgZamqC=b;(Hn6&!vd?UQ8z^QQ@1YiKsz`h+03D){sC3mK>DnH-14-3a1|aow@;Gaazyv>Q1x%gj&qBC*EYis*Czd7(25q?WiG-b=FPdBVqT?893>LC&pXllZSQ(-x4lI#* zeZvtwUu3Q(Jx8}hSz|6GWlfh)REsG#Y*(}f>Ke#vqA3mCf$B2lqBrz39rcbxUv*i+ z{)BDaWS=u&JjTY-<;aay%1N$L!yuTuht}K7%D>-WqE|KwZ=e3=&MLXPSyO+x_U-9J zvcFhKv+Jo+l*CfWs3n>z8%aE862D$a{40|EjLF6qm8Wik*(e(!;0L}4zJANF+9U)I zq6M5X`<@nta9K*3mXHY@dKKsTDrewSkvXXJMfg?d~ zETID9R&fR9=gb7^DaI=MK`hNiLu|2;st+`OitZ1Q?E9ujuibq? zO$wyphhCCM$|#X!%#$_M6xoY}UB4a)?%Z{LirLIPyF!`BqkeA{ORIV0v9Y9b3{uKz2v8{ry1Q+)U!uW zz(dpWGIutA^fYq*ZF5Z21cOp z5!J4Sdgs*~>8hbTnx_SRJ@Us6+RF?rUA7gKx-c}I@swdp$Exfj-Bf*15q%NFpJr4G z#C2xy95cj?R<;2rsNk)y*(O%dBk@12>$+it7TmQR&s5x4^ zt}6PrOKZx|C0BIESW?bdbm4=8+D=v1(`^f><*nRSJ_G5VSf1hKx3;%RnP()q&b8Zb zb?*c{H~F*6-@{QE>~b8p4-bwXa$J$C4!AnU4Y=BX(yG-t{s@V1Cge9qJHZ}is)h0e zjvH}s4&JDwnu9IwIgamveDzJiG=_b?Q#>rRPNhc2m#q8xTY6Pe$4yu4mQI9Ydmva% zO)?cj&_@HoENN2r0A+?s_srX?ak{+S)?{lRXxu2=pUO3%EbPdWy%Fr|mBWXqBeZ2y z9La1OD&Kj6cDmc1E}3<|{}9gdrM$$CJfYS!#cs#z*DJ$bH*cVvWOY2L270V2_yxA} zm7IE5r98pv48?Biw9g;t=nKgDa>F=M8pep~tu;@3repNB_XB>nlk0Wcn!@iQCWuF8 z*eg^{0J}x88=Zr;o+}Dge%|-s9nQD4 zX^t8>PiV~KwAQryUBeO^LF|{rXpS%Qb*Rnylf&(HNrrqved&;wTzY}UtMw+#1kq;~6L*{neHu%`QlY_%MzP^5Wk>BZ6B@ve?FIaBY2{$cZp z*sTod-8$YK-X9n9f|Y}Y)WiN1#`r$058&C6Vd743s?&Z(C)&SukD+Pa2F5OrjT4(C zZg2_)9<4$>1+nCIP`)z4R*Q^kn@XY4pdZpL_FM2Is0ri@P~Ln16I$LEXlLb7Xxge# z3bwo!+%;I{Ch~zgZamqC=b;(Hn6&!vd?UQ8z^QQ@1YiKsz`h+03D){sC3mK>DnH-14-3a1|aow@;Gaazyv>Q1x%gj&qBC*EYis*Czd Date: Tue, 29 Mar 2022 16:06:48 +0200 Subject: [PATCH 2/6] feat(ingest): athena - set Athena location as upstream (#4503) --- .../datahub/ingestion/source/sql/athena.py | 23 ++++++++++--- .../ingestion/source/sql/sql_common.py | 32 +++++++++++++++++-- .../tests/unit/test_athena_source.py | 10 ++++-- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 5f870affbfb67a..66becd9830d756 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -1,4 +1,5 @@ import json +import logging import typing from typing import Dict, List, Optional, Tuple @@ -8,6 +9,7 @@ from datahub.emitter.mcp_builder import DatabaseKey, gen_containers from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.aws.s3_util import make_s3_urn from datahub.ingestion.source.sql.sql_common import ( SQLAlchemyConfig, SQLAlchemySource, @@ -52,7 +54,7 @@ def create(cls, config_dict, ctx): def get_table_properties( self, inspector: Inspector, schema: str, table: str - ) -> Tuple[Optional[str], Optional[Dict[str, str]]]: + ) -> Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]]: if not self.cursor: self.cursor = inspector.dialect._raw_connection(inspector.engine).cursor() @@ -87,7 +89,17 @@ def get_table_properties( metadata.table_type if metadata.table_type else "" ) - return description, custom_properties + location: Optional[str] = custom_properties.get("location", None) + if location is not None: + if location.startswith("s3://"): + location = make_s3_urn(location, self.config.env) + else: + logging.debug( + f"Only s3 url supported for location. Skipping {location}" + ) + location = None + + return description, custom_properties, location # It seems like database/schema filter in the connection string does not work and this to work around that def get_schema_names(self, inspector: Inspector) -> List[str]: @@ -105,10 +117,11 @@ def gen_database_containers( def gen_schema_key(self, db_name: str, schema: str) -> DatabaseKey: return DatabaseKey( - platform=self.platform, - environment=self.config.env, - instance=self.config.platform_instance, database=schema, + platform=self.platform, + instance=self.config.platform_instance + if self.config.platform_instance is not None + else self.config.env, ) def gen_schema_containers( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 8f751cde740b39..f28dc4ee6478bb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -55,6 +55,7 @@ StatefulIngestionSourceBase, ) from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass +from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ( @@ -77,9 +78,11 @@ from datahub.metadata.schema_classes import ( ChangeTypeClass, DataPlatformInstanceClass, + DatasetLineageTypeClass, DatasetPropertiesClass, JobStatusClass, SubTypesClass, + UpstreamClass, ViewPropertiesClass, ) from datahub.telemetry import telemetry @@ -846,13 +849,35 @@ def _process_table( BaseSQLAlchemyCheckpointState, cur_checkpoint.state ) checkpoint_state.add_table_urn(dataset_urn) - description, properties = self.get_table_properties(inspector, schema, table) + + description, properties, location_urn = self.get_table_properties( + inspector, schema, table + ) dataset_properties = DatasetPropertiesClass( name=table, description=description, customProperties=properties, ) dataset_snapshot.aspects.append(dataset_properties) + + if location_urn: + external_upstream_table = UpstreamClass( + dataset=location_urn, + type=DatasetLineageTypeClass.COPY, + ) + lineage_mcpw = MetadataChangeProposalWrapper( + entityType="dataset", + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_snapshot.urn, + aspectName="upstreamLineage", + aspect=UpstreamLineage(upstreams=[external_upstream_table]), + ) + lineage_wu = MetadataWorkUnit( + id=f"{self.platform}-{lineage_mcpw.entityUrn}-{lineage_mcpw.aspectName}", + mcp=lineage_mcpw, + ) + yield lineage_wu + pk_constraints: dict = inspector.get_pk_constraint(table, schema) foreign_keys = self._get_foreign_keys(dataset_urn, inspector, schema, table) schema_fields = self.get_schema_fields(dataset_name, columns, pk_constraints) @@ -896,8 +921,9 @@ def _process_table( def get_table_properties( self, inspector: Inspector, schema: str, table: str - ) -> Tuple[Optional[str], Optional[Dict[str, str]]]: + ) -> Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]]: try: + location: Optional[str] = None # SQLALchemy stubs are incomplete and missing this method. # PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223. table_info: dict = inspector.get_table_comment(table, schema) # type: ignore @@ -918,7 +944,7 @@ def get_table_properties( # The "properties" field is a non-standard addition to SQLAlchemy's interface. properties = table_info.get("properties", {}) - return description, properties + return description, properties, location def get_dataplatform_instance_aspect( self, dataset_urn: str diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py index cd38821f4cb527..812e724eea1716 100644 --- a/metadata-ingestion/tests/unit/test_athena_source.py +++ b/metadata-ingestion/tests/unit/test_athena_source.py @@ -5,6 +5,8 @@ import pytest from freezegun import freeze_time +from src.datahub.ingestion.source.aws.s3_util import make_s3_urn + FROZEN_TIME = "2020-04-14 07:00:00" @@ -56,7 +58,7 @@ def test_athena_get_table_properties(): ], "Parameters": { "comment": "testComment", - "location": "testLocation", + "location": "s3://testLocation", "inputformat": "testInputFormat", "outputformat": "testOutputFormat", "serde.serialization.lib": "testSerde", @@ -74,7 +76,7 @@ def test_athena_get_table_properties(): ctx = PipelineContext(run_id="test") source = AthenaSource(config=config, ctx=ctx) - description, custom_properties = source.get_table_properties( + description, custom_properties, location = source.get_table_properties( inspector=mock_inspector, table=table, schema=schema ) assert custom_properties == { @@ -82,9 +84,11 @@ def test_athena_get_table_properties(): "create_time": "2020-04-14 07:00:00", "inputformat": "testInputFormat", "last_access_time": "2020-04-14 07:00:00", - "location": "testLocation", + "location": "s3://testLocation", "outputformat": "testOutputFormat", "partition_keys": '[{"name": "testKey", "type": "string", "comment": "testComment"}]', "serde.serialization.lib": "testSerde", "table_type": "testType", } + + assert location == make_s3_urn("s3://testLocation", "PROD") From f36cf69360abdf18e7f8d2ad079f1ca11fa8a52f Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 29 Mar 2022 19:40:22 +0530 Subject: [PATCH 3/6] feat(ingest): option for number of workunits in preview (#4517) * feat(ingest): option for number of workunits in preview + documentation update --- metadata-ingestion/README.md | 8 +++++++ .../src/datahub/cli/ingest_cli.py | 15 ++++++++++-- .../src/datahub/ingestion/run/pipeline.py | 23 +++++++++++++++---- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/README.md b/metadata-ingestion/README.md index 3bfb6b3f4c782a..b9bb561794a574 100644 --- a/metadata-ingestion/README.md +++ b/metadata-ingestion/README.md @@ -75,6 +75,14 @@ datahub ingest -c ./examples/recipes/example_to_datahub_rest.yml --preview # Preview with dry-run datahub ingest -c ./examples/recipes/example_to_datahub_rest.yml -n --preview ``` + +By default `--preview` creates 10 workunits. But if you wish to try producing more workunits you can use another option `--preview-workunits` + +```shell +# Preview 20 workunits without sending anything to sink +datahub ingest -c ./examples/recipes/example_to_datahub_rest.yml -n --preview --preview-workunits=20 +``` + ## Transformations If you'd like to modify data before it reaches the ingestion sinks – for instance, adding additional owners or tags – you can use a transformer to write your own module and integrate it with DataHub. diff --git a/metadata-ingestion/src/datahub/cli/ingest_cli.py b/metadata-ingestion/src/datahub/cli/ingest_cli.py index 45f89099bdea5a..7091c5fa3931dd 100644 --- a/metadata-ingestion/src/datahub/cli/ingest_cli.py +++ b/metadata-ingestion/src/datahub/cli/ingest_cli.py @@ -59,6 +59,12 @@ def ingest() -> None: default=False, help="Perform limited ingestion from the source to the sink to get a quick preview.", ) +@click.option( + "--preview-workunits", + type=int, + default=10, + help="The number of workunits to produce for preview.", +) @click.option( "--strict-warnings/--no-strict-warnings", default=False, @@ -68,7 +74,12 @@ def ingest() -> None: @telemetry.with_telemetry @memory_leak_detector.with_leak_detection def run( - ctx: click.Context, config: str, dry_run: bool, preview: bool, strict_warnings: bool + ctx: click.Context, + config: str, + dry_run: bool, + preview: bool, + strict_warnings: bool, + preview_workunits: int, ) -> None: """Ingest metadata into DataHub.""" @@ -79,7 +90,7 @@ def run( try: logger.debug(f"Using config: {pipeline_config}") - pipeline = Pipeline.create(pipeline_config, dry_run, preview) + pipeline = Pipeline.create(pipeline_config, dry_run, preview, preview_workunits) except ValidationError as e: click.echo(e, err=True) sys.exit(1) diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py index b2ebc3d984facc..7835c1c3de832f 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py @@ -103,11 +103,16 @@ class Pipeline: transformers: List[Transformer] def __init__( - self, config: PipelineConfig, dry_run: bool = False, preview_mode: bool = False + self, + config: PipelineConfig, + dry_run: bool = False, + preview_mode: bool = False, + preview_workunits: int = 10, ): self.config = config self.dry_run = dry_run self.preview_mode = preview_mode + self.preview_workunits = preview_workunits self.ctx = PipelineContext( run_id=self.config.run_id, datahub_api=self.config.datahub_api, @@ -169,17 +174,27 @@ def _configure_reporting(self) -> None: @classmethod def create( - cls, config_dict: dict, dry_run: bool = False, preview_mode: bool = False + cls, + config_dict: dict, + dry_run: bool = False, + preview_mode: bool = False, + preview_workunits: int = 10, ) -> "Pipeline": config = PipelineConfig.parse_obj(config_dict) - return cls(config, dry_run=dry_run, preview_mode=preview_mode) + return cls( + config, + dry_run=dry_run, + preview_mode=preview_mode, + preview_workunits=preview_workunits, + ) def run(self) -> None: callback = LoggingCallback() extractor: Extractor = self.extractor_class() for wu in itertools.islice( - self.source.get_workunits(), 10 if self.preview_mode else None + self.source.get_workunits(), + self.preview_workunits if self.preview_mode else None, ): # TODO: change extractor interface extractor.configure({}, self.ctx) From d30b6e1465e928daf74cd62882408243f7b8d246 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 29 Mar 2022 19:45:36 +0530 Subject: [PATCH 4/6] feat(ingest): Add config to improve user exp for initial ingestion and fix docs (#4510) * feat(snowflake): change defaults to improve user experience for initial ingestion + documentation --- metadata-ingestion/source_docs/bigquery.md | 24 +++++++++++++++---- metadata-ingestion/source_docs/snowflake.md | 2 ++ .../datahub/ingestion/source/sql/snowflake.py | 24 ++++++++++++++----- .../ingestion/source/usage/snowflake_usage.py | 3 ++- .../ingestion/source_config/sql/snowflake.py | 3 ++- .../ingestion/source_report/sql/snowflake.py | 7 +++++- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/metadata-ingestion/source_docs/bigquery.md b/metadata-ingestion/source_docs/bigquery.md index 5d72b5cef43e21..9a9cc14f5f8090 100644 --- a/metadata-ingestion/source_docs/bigquery.md +++ b/metadata-ingestion/source_docs/bigquery.md @@ -101,6 +101,19 @@ source: # Coordinates project_id: my_project_id + # `schema_pattern` for BQ Datasets + schema_pattern: + allow: + - finance_bq_dataset + + table_pattern: + deny: + # The exact name of the table is revenue_table_name + # The reason we have this `.*` at the beginning is because the current implmenetation of table_pattern is testing + # project_id.dataset_name.table_name + # We will improve this in the future + - .*revenue_table_name + sink: # sink configs ``` @@ -123,8 +136,8 @@ As a SQL-based service, the Athena integration is also supported by our SQL prof | `table_pattern.allow` | | | List of regex patterns for tables to include in ingestion. | | `table_pattern.deny` | | | List of regex patterns for tables to exclude from ingestion. | | `table_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | -| `schema_pattern.allow` | | | List of regex patterns for schemas to include in ingestion. | -| `schema_pattern.deny` | | | List of regex patterns for schemas to exclude from ingestion. | +| `schema_pattern.allow` | | | List of regex patterns for BigQuery dataset to include in ingestion. | +| `schema_pattern.deny` | | | List of regex patterns for BigQuery dataset to exclude from ingestion. | | `schema_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | | `view_pattern.allow` | | | List of regex patterns for views to include in ingestion. | | `view_pattern.deny` | | | List of regex patterns for views to exclude from ingestion. | @@ -139,10 +152,11 @@ As a SQL-based service, the Athena integration is also supported by our SQL prof | `use_exported_bigquery_audit_metadata` | | `False` | When configured, use `BigQueryAuditMetadata` in `bigquery_audit_metadata_datasets` to compute lineage information. | | `use_date_sharded_audit_log_tables` | | `False` | Whether to read date sharded tables or time partitioned tables when extracting lineage from exported audit logs. | | `bigquery_audit_metadata_datasets` | | None | A list of datasets that contain a table named `cloudaudit_googleapis_com_data_access` which contain BigQuery audit logs, specifically, those containing `BigQueryAuditMetadata`. It is recommended that the project of the dataset is also specified, for example, `projectA.datasetB`. | -| `domain.domain_key.allow` | | | List of regex patterns for tables/schemas to set domain_key domain key (domain_key can be any string like `sales`. There can be multiple domain key specified. | -| `domain.domain_key.deny` | | | List of regex patterns for tables/schemas to not assign domain_key. There can be multiple domain key specified. | +| `domain.domain_key.allow` | | | List of regex patterns for tables/BigQuery dataset to set domain_key domain key (domain_key can be any string like `sales`. There can be multiple domain key specified. | +| `domain.domain_key.deny` | | | List of regex patterns for tables/BigQuery dataset to not assign domain_key. There can be multiple domain key specified. | | `domain.domain_key.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching.There can be multiple domain key specified. | -| `lineage_client_project_id` | | None | The project to use when creating the BigQuery Client. If left empty, the required `project_id` will be used. | +| `lineage_client_project_id` | | None | The project to use when creating the BigQuery Client. If left empty, the required `project_id` will be used. This is helpful in case the default project_id is not used for querying. | +| `use_v2_audit_metadata` | | `False` | Whether to use `BigQuery audit logs` to get the lineage or not | The following parameters are only relevant if include_table_lineage is set to true: diff --git a/metadata-ingestion/source_docs/snowflake.md b/metadata-ingestion/source_docs/snowflake.md index 1fea1c6342b695..8702bc54ff15cb 100644 --- a/metadata-ingestion/source_docs/snowflake.md +++ b/metadata-ingestion/source_docs/snowflake.md @@ -153,6 +153,8 @@ Note that a `.` is used to denote nested fields in the YAML recipe. | `include_table_lineage` | | `True` | If enabled, populates the snowflake table-to-table and s3-to-snowflake table lineage. Requires appropriate grants given to the role. | | `include_view_lineage` | | `True` | If enabled, populates the snowflake view->table and table->view lineages (no view->view lineage yet). Requires appropriate grants given to the role, and `include_table_lineage` to be `True`. | | `bucket_duration` | | `"DAY"` | Duration to bucket lineage data extraction by. Can be `"DAY"` or `"HOUR"`. | +| `report_upstream_lineage` | | `False` | Whether to report upstream lineage in the report. This should be marked as `True` in case someone is debugging lineage ingestion issues | +| `ignore_start_time_lineage` | | `False` | Whether to ignore `start_time` and read all data for lineage. It is meant to be used for initial ingestion | | `start_time` | | Start of last full day in UTC (or hour, depending on `bucket_duration`) | Earliest time of lineage data to consider. For the bootstrap run, set it as far back in time as possible. | | `end_time` | | End of last full day in UTC (or hour, depending on `bucket_duration`) | Latest time of lineage data to consider. | | `profiling` | | See the defaults for [profiling config](./sql_profiles.md#Config-details). | See [profiling config](./sql_profiles.md#Config-details). | diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py index ab454ee20cd3eb..a05ff5c73319ef 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py @@ -246,7 +246,9 @@ def _populate_view_downstream_lineage( query_start_time DESC ) = 1 """.format( - start_time_millis=int(self.config.start_time.timestamp() * 1000), + start_time_millis=int(self.config.start_time.timestamp() * 1000) + if not self.config.ignore_start_time_lineage + else 0, end_time_millis=int(self.config.end_time.timestamp() * 1000), ) @@ -333,7 +335,9 @@ def _populate_external_lineage(self) -> None: FROM external_table_lineage_history WHERE downstream_table_domain = 'Table' QUALIFY ROW_NUMBER() OVER (PARTITION BY downstream_table_name ORDER BY query_start_time DESC) = 1""".format( - start_time_millis=int(self.config.start_time.timestamp() * 1000), + start_time_millis=int(self.config.start_time.timestamp() * 1000) + if not self.config.ignore_start_time_lineage + else 0, end_time_millis=int(self.config.end_time.timestamp() * 1000), ) @@ -401,7 +405,9 @@ def _populate_lineage(self) -> None: FROM table_lineage_history WHERE upstream_table_domain in ('Table', 'External table') and downstream_table_domain = 'Table' QUALIFY ROW_NUMBER() OVER (PARTITION BY downstream_table_name, upstream_table_name ORDER BY query_start_time DESC) = 1 """.format( - start_time_millis=int(self.config.start_time.timestamp() * 1000), + start_time_millis=int(self.config.start_time.timestamp() * 1000) + if not self.config.ignore_start_time_lineage + else 0, end_time_millis=int(self.config.end_time.timestamp() * 1000), ) num_edges: int = 0 @@ -503,14 +509,20 @@ def _get_upstream_lineage_info( logger.debug( f"Upstream lineage of '{dataset_name}': {[u.dataset for u in upstream_tables]}" ) - self.report.upstream_lineage[dataset_name] = [ - u.dataset for u in upstream_tables - ] + if self.config.report_upstream_lineage: + self.report.upstream_lineage[dataset_name] = [ + u.dataset for u in upstream_tables + ] return UpstreamLineage(upstreams=upstream_tables), column_lineage return None def add_config_to_report(self): self.report.cleaned_host_port = self.config.host_port + self.report.ignore_start_time_lineage = self.config.ignore_start_time_lineage + self.report.report_upstream_lineage = self.config.report_upstream_lineage + if not self.report.ignore_start_time_lineage: + self.report.lineage_start_time = self.config.start_time + self.report.lineage_end_time = self.config.end_time if self.config.provision_role is not None: self.report.run_ingestion = self.config.provision_role.run_ingestion diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py index 1789f29de537fc..da0ccce26eacab 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py @@ -274,7 +274,8 @@ def _make_usage_query(self) -> str: start_time = int(self.config.start_time.timestamp() * 1000) end_time = int(self.config.end_time.timestamp() * 1000) return SNOWFLAKE_USAGE_SQL_TEMPLATE.format( - start_time_millis=start_time, end_time_millis=end_time + start_time_millis=start_time, + end_time_millis=end_time, ) def _make_sql_engine(self) -> Engine: diff --git a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py index 87477cbbe86907..5289e0ab6ceb39 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py @@ -83,7 +83,6 @@ def host_port_is_valid(cls, v, values, **kwargs): v = remove_protocol(v) v = remove_trailing_slashes(v) v = remove_suffix(v, ".snowflakecomputing.com") - logger.info(f"Cleaned Host port is {v}") return v @pydantic.validator("authentication_type", always=True) @@ -186,6 +185,8 @@ class SnowflakeConfig(BaseSnowflakeConfig, SQLAlchemyConfig): database: Optional[str] # deprecated provision_role: Optional[SnowflakeProvisionRoleConfig] = None + ignore_start_time_lineage: bool = False + report_upstream_lineage: bool = False @pydantic.validator("database") def note_database_opt_deprecation(cls, v, values, **kwargs): diff --git a/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py index 81b1a95d682708..1d53b6d04d4ebd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source_report/sql/snowflake.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field -from typing import Dict, List +from datetime import datetime +from typing import Dict, List, Optional from datahub.ingestion.source.sql.sql_common import SQLSourceReport from datahub.ingestion.source_report.time_window import BaseTimeWindowReport @@ -16,7 +17,11 @@ class SnowflakeReport(BaseSnowflakeReport, SQLSourceReport): num_table_to_view_edges_scanned: int = 0 num_view_to_table_edges_scanned: int = 0 num_external_table_edges_scanned: int = 0 + ignore_start_time_lineage: Optional[bool] = None + report_upstream_lineage: Optional[bool] = None upstream_lineage: Dict[str, List[str]] = field(default_factory=dict) + lineage_start_time: Optional[datetime] = None + lineage_end_time: Optional[datetime] = None cleaned_host_port: str = "" run_ingestion: bool = False From 4f7d0f328193f394610c3fb429ddfef708ba8523 Mon Sep 17 00:00:00 2001 From: Ravindra Lanka Date: Tue, 29 Mar 2022 09:42:43 -0700 Subject: [PATCH 5/6] Fix: Snowflake Table to View lineage (#4520) --- .../datahub/ingestion/source/sql/snowflake.py | 82 ++++++------------- 1 file changed, 27 insertions(+), 55 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py b/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py index a05ff5c73319ef..34032e67745f9e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/snowflake.py @@ -187,9 +187,7 @@ def _populate_view_upstream_lineage(self, engine: sqlalchemy.engine.Engine) -> N def _populate_view_downstream_lineage( self, engine: sqlalchemy.engine.Engine ) -> None: - # NOTE: This query captures both the upstream and downstream table lineage for views. - # We need this query to populate the downstream lineage of a view, - # as well as to delete the false direct edges between the upstream and downstream tables of a view. + # This query captures the downstream table lineage for views. # See https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html#usage-notes for current limitations on capturing the lineage for views. # Eg: For viewA->viewB->ViewC->TableD, snowflake does not yet log intermediate view logs, resulting in only the viewA->TableD edge. view_lineage_query: str = """ @@ -198,9 +196,6 @@ def _populate_view_downstream_lineage( vu.value : "objectName" AS view_name, vu.value : "objectDomain" AS view_domain, vu.value : "columns" AS view_columns, - r.value : "objectName" AS upstream_table_name, - r.value : "objectDomain" AS upstream_table_domain, - r.value : "columns" AS upstream_table_columns, w.value : "objectName" AS downstream_table_name, w.value : "objectDomain" AS downstream_table_domain, w.value : "columns" AS downstream_table_columns, @@ -213,34 +208,26 @@ def _populate_view_downstream_lineage( snowflake.account_usage.access_history ) t, lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) vu, - lateral flatten(input => t.BASE_OBJECTS_ACCESSED) r, lateral flatten(input => t.OBJECTS_MODIFIED) w WHERE vu.value : "objectId" IS NOT NULL - AND r.value : "objectId" IS NOT NULL AND w.value : "objectId" IS NOT NULL + AND w.value : "objectName" NOT LIKE '%.GE_TMP_%' + AND w.value : "objectName" NOT LIKE '%.GE_TEMP_%' AND t.query_start_time >= to_timestamp_ltz({start_time_millis}, 3) AND t.query_start_time < to_timestamp_ltz({end_time_millis}, 3) ) SELECT view_name, view_columns, - upstream_table_name, - upstream_table_domain, - upstream_table_columns, downstream_table_name, - downstream_table_domain, downstream_table_columns FROM view_lineage_history WHERE view_domain in ('View', 'Materialized view') - AND view_name != upstream_table_name - AND upstream_table_name != downstream_table_name - AND view_name != downstream_table_name QUALIFY ROW_NUMBER() OVER ( PARTITION BY view_name, - upstream_table_name, downstream_table_name ORDER BY query_start_time DESC @@ -253,45 +240,10 @@ def _populate_view_downstream_lineage( ) assert self._lineage_map is not None - num_edges: int = 0 - num_false_edges: int = 0 + self.report.num_view_to_table_edges_scanned = 0 try: - for db_row in engine.execute(view_lineage_query): - # We get two edges here. - # (1) False UpstreamTable->Downstream table that will be deleted. - # (2) View->DownstreamTable that will be added. - - view_name: str = db_row[0].lower().replace('"', "") - upstream_table: str = db_row[2].lower().replace('"', "") - downstream_table: str = db_row[5].lower().replace('"', "") - # (1) Delete false direct edge between upstream_table and downstream_table - prior_edges: List[Tuple[str, str, str]] = self._lineage_map[ - downstream_table - ] - self._lineage_map[downstream_table] = [ - entry - for entry in self._lineage_map[downstream_table] - if entry[0] != upstream_table - ] - for false_edge in set(prior_edges) - set( - self._lineage_map[downstream_table] - ): - logger.debug( - f"False Table->Table edge removed: Lineage[Table(Down)={downstream_table}]:Table(Up)={false_edge}." - ) - num_false_edges += 1 - - # (2) Add view->downstream table lineage. - self._lineage_map[downstream_table].append( - # (, , ) - (view_name, db_row[1], db_row[7]) - ) - logger.debug( - f"View->Table: Lineage[Table(Down)={downstream_table}]:View(Up)={self._lineage_map[downstream_table]}, downstream_domain={db_row[6]}" - ) - num_edges += 1 - + db_rows = engine.execute(view_lineage_query) except Exception as e: self.warn( logger, @@ -299,10 +251,30 @@ def _populate_view_downstream_lineage( f"Extracting the view lineage from Snowflake failed." f"Please check your permissions. Continuing...\nError was {e}.", ) + else: + for db_row in db_rows: + view_name: str = db_row["view_name"].lower().replace('"', "") + downstream_table: str = ( + db_row["downstream_table_name"].lower().replace('"', "") + ) + # Capture view->downstream table lineage. + self._lineage_map[downstream_table].append( + # (, , ) + ( + view_name, + db_row["view_columns"], + db_row["downstream_table_columns"], + ) + ) + self.report.num_view_to_table_edges_scanned += 1 + + logger.debug( + f"View->Table: Lineage[Table(Down)={downstream_table}]:View(Up)={self._lineage_map[downstream_table]}" + ) + logger.info( - f"Found {num_edges} View->Table edges. Removed {num_false_edges} false Table->Table edges." + f"Found {self.report.num_view_to_table_edges_scanned} View->Table edges." ) - self.report.num_view_to_table_edges_scanned = num_edges def _populate_view_lineage(self) -> None: if not self.config.include_view_lineage: From df69e762895b39d3d06d40e7a7965a057f76d32a Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Tue, 29 Mar 2022 12:02:22 -0700 Subject: [PATCH 6/6] feat(policy): Add domain based and view based policies (#4516) * Add domain based policy and view based policy * Add new files * Fix issue * Fix comment on markdown * Fix to John's comments * Update policies.md * Remove comments * Fix to John's comments * Fix policy details page * Add back notebook privileges * Fix graphql Co-authored-by: John Joyce --- .../datahub/graphql/GmsGraphQLEngine.java | 11 +- .../authorization/AuthorizationUtils.java | 22 +- .../load/TimeSeriesAspectResolver.java | 23 + .../policy/GetGrantedPrivilegesResolver.java | 53 ++ .../mappers/PolicyInfoPolicyMapper.java | 44 +- .../mappers/PolicyUpdateInputInfoMapper.java | 36 +- .../graphql/types/usage/UsageType.java | 27 +- .../src/main/resources/entity.graphql | 168 ++++- .../app/authorization/UnauthorizedPage.tsx | 11 + .../src/app/entity/EntityPage.tsx | 58 +- .../src/app/policy/PoliciesPage.tsx | 56 +- .../src/app/policy/PolicyBuilderModal.tsx | 4 +- .../src/app/policy/PolicyDetailsModal.tsx | 70 +- .../src/app/policy/PolicyPrivilegeForm.tsx | 334 ++++++---- .../src/app/policy/policyUtils.ts | 89 ++- datahub-web-react/src/graphql/policy.graphql | 18 + docs/policies.md | 68 +- .../metadata/entity/EntityService.java | 24 +- .../metadata/entity/ebean/EbeanAspectDao.java | 3 + .../entity/ebean/EbeanEntityService.java | 2 +- .../linkedin/policy/DataHubResourceFilter.pdl | 10 +- .../linkedin/policy/PolicyMatchCondition.pdl | 11 + .../linkedin/policy/PolicyMatchCriterion.pdl | 23 + .../com/linkedin/policy/PolicyMatchFilter.pdl | 13 + .../authorization/AuthorizationRequest.java | 58 +- .../datahub/authorization/ResourceSpec.java | 25 +- .../authorization/AuthorizationManager.java | 51 +- .../datahub/authorization/FieldResolver.java | 53 ++ .../datahub/authorization/FilterUtils.java | 68 ++ .../datahub/authorization/PolicyEngine.java | 221 ++++--- .../authorization/ResolvedResourceSpec.java | 33 + .../authorization/ResourceSpecResolver.java | 36 + .../DomainFieldResolverProvider.java | 58 ++ .../EntityTypeFieldResolverProvider.java | 21 + .../EntityUrnFieldResolverProvider.java | 21 + .../OwnerFieldResolverProvider.java | 58 ++ .../ResourceFieldResolverProvider.java | 21 + .../ResourceFieldType.java | 23 + .../AuthorizationManagerTest.java | 42 +- .../authorization/PolicyEngineTest.java | 623 ++++++++++-------- .../auth/AuthorizationManagerFactory.java | 5 +- .../boot/steps/IngestPoliciesStep.java | 17 +- ...com.linkedin.entity.entities.snapshot.json | 48 +- .../linkedin/entity/client/EntityClient.java | 7 + .../entity/client/JavaEntityClient.java | 12 + .../entity/client/OwnershipClient.java | 59 -- .../entity/client/RestliEntityClient.java | 13 +- .../resources/entity/EntityV2Resource.java | 9 +- .../war/src/main/resources/boot/policies.json | 202 +----- .../java/com/linkedin/metadata/Constants.java | 1 + .../authorization/PoliciesConfig.java | 34 +- 51 files changed, 2068 insertions(+), 929 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java create mode 100644 datahub-web-react/src/app/authorization/UnauthorizedPage.tsx create mode 100644 metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCondition.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCriterion.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchFilter.pdl create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/FieldResolver.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResourceSpecResolver.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java create mode 100644 metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldType.java delete mode 100644 metadata-service/restli-client/src/main/java/com/linkedin/entity/client/OwnershipClient.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index b00a885d57ccec..4280a5ec2b8788 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -45,6 +45,7 @@ import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties; import com.linkedin.datahub.graphql.generated.Notebook; import com.linkedin.datahub.graphql.generated.Owner; +import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue; import com.linkedin.datahub.graphql.generated.RecommendationContent; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResult; import com.linkedin.datahub.graphql.generated.SearchResult; @@ -73,6 +74,7 @@ import com.linkedin.datahub.graphql.resolvers.group.ListGroupsResolver; import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupMembersResolver; import com.linkedin.datahub.graphql.resolvers.group.RemoveGroupResolver; +import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.CancelIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.CreateIngestionExecutionRequestResolver; import com.linkedin.datahub.graphql.resolvers.ingest.execution.GetIngestionExecutionRequestResolver; @@ -606,6 +608,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { (env) -> env.getArgument(URN_FIELD_NAME)))) .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient)) + .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver()) .dataFetcher("listUsers", new ListUsersResolver(this.entityClient)) .dataFetcher("listGroups", @@ -751,8 +754,14 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder new EntityTypeBatchResolver( new ArrayList<>(entityTypes), (env) -> ((AutoCompleteResultForEntity) env.getSource()).getEntities())) + ) + .type("PolicyMatchCriterionValue", typeWiring -> typeWiring + .dataFetcher("entity", new AuthenticatedResolver<>( + new EntityTypeResolver( + new ArrayList<>(entityTypes), + (env) -> ((PolicyMatchCriterionValue) env.getSource()).getEntity())) + ) ); - ; } /** diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index 490d5f779b0e6e..4c52f62115b73b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -14,25 +14,25 @@ public class AuthorizationUtils { public static boolean canManageUsersAndGroups(@Nonnull QueryContext context) { - final Authorizer authorizer = context.getAuthorizer(); - final String actor = context.getActorUrn(); - final ConjunctivePrivilegeGroup andGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.MANAGE_USERS_AND_GROUPS_PRIVILEGE.getType())); - return isAuthorized(authorizer, actor, new DisjunctivePrivilegeGroup(ImmutableList.of(andGroup))); + return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_USERS_AND_GROUPS_PRIVILEGE); } public static boolean canGeneratePersonalAccessToken(@Nonnull QueryContext context) { - final Authorizer authorizer = context.getAuthorizer(); - final String actor = context.getActorUrn(); - final ConjunctivePrivilegeGroup andGroup = new ConjunctivePrivilegeGroup( - ImmutableList.of(PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE.getType())); - return isAuthorized(authorizer, actor, new DisjunctivePrivilegeGroup(ImmutableList.of(andGroup))); + return isAuthorized(context, Optional.empty(), PoliciesConfig.GENERATE_PERSONAL_ACCESS_TOKENS_PRIVILEGE); } public static boolean canManageDomains(@Nonnull QueryContext context) { + return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE); + } + + public static boolean isAuthorized( + @Nonnull QueryContext context, + @Nonnull Optional resourceSpec, + @Nonnull PoliciesConfig.Privilege privilege) { final Authorizer authorizer = context.getAuthorizer(); final String actor = context.getActorUrn(); - final ConjunctivePrivilegeGroup andGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(PoliciesConfig.MANAGE_DOMAINS_PRIVILEGE.getType())); - return isAuthorized(authorizer, actor, new DisjunctivePrivilegeGroup(ImmutableList.of(andGroup))); + final ConjunctivePrivilegeGroup andGroup = new ConjunctivePrivilegeGroup(ImmutableList.of(privilege.getType())); + return isAuthorized(authorizer, actor, resourceSpec, new DisjunctivePrivilegeGroup(ImmutableList.of(andGroup))); } public static boolean isAuthorized( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java index 35c6db088292eb..f40214b8eede75 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java @@ -1,14 +1,20 @@ package com.linkedin.datahub.graphql.resolvers.load; +import com.datahub.authorization.ResourceSpec; import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.TimeSeriesAspect; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.EnvelopedAspect; +import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; @@ -42,6 +48,18 @@ public TimeSeriesAspectResolver(final EntityClient client, final String entityNa _aspectMapper = aspectMapper; } + /** + * Check whether the actor is authorized to fetch the timeseries aspect given the resource urn + */ + private boolean isAuthorized(QueryContext context, String urn) { + if (_entityName.equals(Constants.DATASET_ENTITY_NAME) && _aspectName.equals( + Constants.DATASET_PROFILE_ASPECT_NAME)) { + return AuthorizationUtils.isAuthorized(context, Optional.of(new ResourceSpec(_entityName, urn)), + PoliciesConfig.VIEW_DATASET_PROFILE_PRIVILEGE); + } + return true; + } + @Override public CompletableFuture> get(DataFetchingEnvironment environment) { return CompletableFuture.supplyAsync(() -> { @@ -50,6 +68,11 @@ public CompletableFuture> get(DataFetchingEnvironment env // Fetch the urn, assuming the parent has an urn field. // todo: what if the parent urn isn't projected? final String urn = ((Entity) environment.getSource()).getUrn(); + + if (!isAuthorized(context, urn)) { + return Collections.emptyList(); + } + final Long maybeStartTimeMillis = environment.getArgumentOrDefault("startTimeMillis", null); final Long maybeEndTimeMillis = environment.getArgumentOrDefault("endTimeMillis", null); // Max number of aspects to return. diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java new file mode 100644 index 00000000000000..0d22256bfc199e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java @@ -0,0 +1,53 @@ +package com.linkedin.datahub.graphql.resolvers.policy; + +import com.datahub.authorization.AuthorizationManager; +import com.datahub.authorization.ResourceSpec; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.GetGrantedPrivilegesInput; +import com.linkedin.datahub.graphql.generated.Privileges; +import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; + + +/** + * Resolver to support the getGrantedPrivileges end point + * Fetches all privileges that are granted for the given actor for the given resource (optional) + */ +public class GetGrantedPrivilegesResolver implements DataFetcher> { + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final GetGrantedPrivilegesInput input = + bindArgument(environment.getArgument("input"), GetGrantedPrivilegesInput.class); + final String actor = input.getActorUrn(); + if (!isAuthorized(context, actor)) { + throw new AuthorizationException("Unauthorized to get privileges for the given author."); + } + final Optional resourceSpec = Optional.ofNullable(input.getResourceSpec()) + .map(spec -> new ResourceSpec(EntityTypeMapper.getName(spec.getResourceType()), spec.getResourceUrn())); + + if (context.getAuthorizer() instanceof AuthorizationManager) { + AuthorizationManager authorizationManager = (AuthorizationManager) context.getAuthorizer(); + List privileges = authorizationManager.getGrantedPrivileges(actor, resourceSpec); + return CompletableFuture.supplyAsync(() -> Privileges.builder() + .setPrivileges(privileges) + .build()); + } + throw new UnsupportedOperationException( + String.format("GetGrantedPrivileges function is not supported on authorizer of type %s", + context.getAuthorizer().getClass().getSimpleName())); + } + + private boolean isAuthorized(final QueryContext context, final String actor) { + return actor.equals(context.getActorUrn()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyInfoPolicyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyInfoPolicyMapper.java index 27793931406f0c..b23d70a53d88f7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyInfoPolicyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyInfoPolicyMapper.java @@ -3,13 +3,19 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.ActorFilter; import com.linkedin.datahub.graphql.generated.Policy; +import com.linkedin.datahub.graphql.generated.PolicyMatchCondition; +import com.linkedin.datahub.graphql.generated.PolicyMatchCriterion; +import com.linkedin.datahub.graphql.generated.PolicyMatchCriterionValue; +import com.linkedin.datahub.graphql.generated.PolicyMatchFilter; import com.linkedin.datahub.graphql.generated.PolicyState; import com.linkedin.datahub.graphql.generated.PolicyType; import com.linkedin.datahub.graphql.generated.ResourceFilter; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; import com.linkedin.policy.DataHubResourceFilter; +import java.net.URISyntaxException; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -17,7 +23,7 @@ /** * Maps {@link com.linkedin.policy.DataHubPolicyInfo} to GraphQL {@link com.linkedin.datahub.graphql.generated.Policy}. */ -public class PolicyInfoPolicyMapper implements ModelMapper { +public class PolicyInfoPolicyMapper implements ModelMapper { public static final PolicyInfoPolicyMapper INSTANCE = new PolicyInfoPolicyMapper(); @@ -48,16 +54,10 @@ private ActorFilter mapActors(final DataHubActorFilter actorFilter) { result.setAllUsers(actorFilter.isAllUsers()); result.setResourceOwners(actorFilter.isResourceOwners()); if (actorFilter.hasGroups()) { - result.setGroups(actorFilter.getGroups() - .stream() - .map(Urn::toString) - .collect(Collectors.toList())); + result.setGroups(actorFilter.getGroups().stream().map(Urn::toString).collect(Collectors.toList())); } if (actorFilter.hasUsers()) { - result.setUsers(actorFilter.getUsers() - .stream() - .map(Urn::toString) - .collect(Collectors.toList())); + result.setUsers(actorFilter.getUsers().stream().map(Urn::toString).collect(Collectors.toList())); } return result; } @@ -71,7 +71,33 @@ private ResourceFilter mapResources(final DataHubResourceFilter resourceFilter) if (resourceFilter.hasResources()) { result.setResources(resourceFilter.getResources()); } + if (resourceFilter.hasFilter()) { + result.setFilter(mapFilter(resourceFilter.getFilter())); + } return result; } + private PolicyMatchFilter mapFilter(final com.linkedin.policy.PolicyMatchFilter filter) { + return PolicyMatchFilter.builder() + .setCriteria(filter.getCriteria() + .stream() + .map(criterion -> PolicyMatchCriterion.builder() + .setField(criterion.getField()) + .setValues(criterion.getValues().stream().map(this::mapValue).collect(Collectors.toList())) + .setCondition(PolicyMatchCondition.valueOf(criterion.getCondition().name())) + .build()) + .collect(Collectors.toList())) + .build(); + } + + private PolicyMatchCriterionValue mapValue(final String value) { + try { + // If value is urn, set entity field + Urn urn = Urn.createFromString(value); + return PolicyMatchCriterionValue.builder().setValue(value).setEntity(UrnToEntityMapper.map(urn)).build(); + } catch (URISyntaxException e) { + // Value is not an urn. Just set value + return PolicyMatchCriterionValue.builder().setValue(value).build(); + } + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyUpdateInputInfoMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyUpdateInputInfoMapper.java index 548992186dd14d..1e0f41c68b32e9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyUpdateInputInfoMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/mappers/PolicyUpdateInputInfoMapper.java @@ -4,16 +4,22 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.StringArray; import com.linkedin.datahub.graphql.generated.ActorFilterInput; +import com.linkedin.datahub.graphql.generated.PolicyMatchFilterInput; import com.linkedin.datahub.graphql.generated.PolicyUpdateInput; import com.linkedin.datahub.graphql.generated.ResourceFilterInput; import com.linkedin.datahub.graphql.types.mappers.ModelMapper; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; import com.linkedin.policy.DataHubResourceFilter; +import com.linkedin.policy.PolicyMatchCondition; +import com.linkedin.policy.PolicyMatchCriterion; +import com.linkedin.policy.PolicyMatchCriterionArray; +import com.linkedin.policy.PolicyMatchFilter; import java.net.URISyntaxException; import java.util.stream.Collectors; import javax.annotation.Nonnull; + /** * Maps GraphQL {@link PolicyUpdateInput} to DataHub backend {@link DataHubPolicyInfo}. */ @@ -46,32 +52,42 @@ private DataHubActorFilter mapActors(final ActorFilterInput actorInput) { result.setAllUsers(actorInput.getAllUsers()); result.setResourceOwners(actorInput.getResourceOwners()); if (actorInput.getGroups() != null) { - result.setGroups(new UrnArray(actorInput.getGroups() - .stream() - .map(this::createUrn) - .collect(Collectors.toList()))); + result.setGroups(new UrnArray(actorInput.getGroups().stream().map(this::createUrn).collect(Collectors.toList()))); } if (actorInput.getUsers() != null) { - result.setUsers(new UrnArray(actorInput.getUsers() - .stream() - .map(this::createUrn) - .collect(Collectors.toList()))); + result.setUsers(new UrnArray(actorInput.getUsers().stream().map(this::createUrn).collect(Collectors.toList()))); } return result; } private DataHubResourceFilter mapResources(final ResourceFilterInput resourceInput) { final DataHubResourceFilter result = new DataHubResourceFilter(); + if (resourceInput.getAllResources() != null) { + result.setAllResources(resourceInput.getAllResources()); + } // This is an implicit mapping between GQL EntityType and Entity Name as known by GMS. // Be careful about maintaining this contract. - result.setType(resourceInput.getType()); - result.setAllResources(resourceInput.getAllResources()); + if (resourceInput.getType() != null) { + result.setType(resourceInput.getType()); + } if (resourceInput.getResources() != null) { result.setResources(new StringArray(resourceInput.getResources())); } + if (resourceInput.getFilter() != null) { + result.setFilter(mapFilter(resourceInput.getFilter())); + } return result; } + private PolicyMatchFilter mapFilter(final PolicyMatchFilterInput filter) { + return new PolicyMatchFilter().setCriteria(new PolicyMatchCriterionArray(filter.getCriteria() + .stream() + .map(criterion -> new PolicyMatchCriterion().setField(criterion.getField()) + .setValues(new StringArray(criterion.getValues())) + .setCondition(PolicyMatchCondition.valueOf(criterion.getCondition().name()))) + .collect(Collectors.toList()))); + } + private Urn createUrn(String urnStr) { try { return Urn.createFromString(urnStr); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java index d294b9d7951b3c..658c8791a77717 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/usage/UsageType.java @@ -1,25 +1,33 @@ package com.linkedin.datahub.graphql.types.usage; +import com.datahub.authorization.ResourceSpec; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; - import com.linkedin.datahub.graphql.UsageStatsKey; import com.linkedin.datahub.graphql.VersionedAspectKey; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.usage.UsageClient; import com.linkedin.usage.UsageQueryResult; import graphql.execution.DataFetcherResult; import java.net.URISyntaxException; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class UsageType { private final UsageClient _usageClient; public UsageType(final UsageClient usageClient) { _usageClient = usageClient; } + /** * Retrieves an list of aspects given a list of {@link VersionedAspectKey} structs. The list returned is expected to * be of same length of the list of keys, where nulls are provided in place of an aspect object if an entity cannot be found. @@ -27,15 +35,22 @@ public UsageType(final UsageClient usageClient) { * @param context the {@link QueryContext} corresponding to the request. */ public List> batchLoad( - @Nonnull List keys, @Nonnull QueryContext context - ) throws Exception { + @Nonnull List keys, @Nonnull QueryContext context) throws Exception { try { return keys.stream().map(key -> { + Urn resourceUrn = UrnUtils.getUrn(key.getResource()); + if (!AuthorizationUtils.isAuthorized(context, + Optional.of(new ResourceSpec(resourceUrn.getEntityType(), key.getResource())), + PoliciesConfig.VIEW_DATASET_USAGE_PRIVILEGE)) { + log.debug("User {} is not authorized to view usage information for dataset {}", context.getActorUrn(), + key.getResource()); + return DataFetcherResult.newResult().build(); + } try { - UsageQueryResult usageQueryResult = _usageClient.getUsageStats(key.getResource(), key.getRange(), context.getAuthentication()); + UsageQueryResult usageQueryResult = + _usageClient.getUsageStats(key.getResource(), key.getRange(), context.getAuthentication()); return DataFetcherResult.newResult().data( - UsageQueryResultMapper.map(usageQueryResult) - ).build(); + UsageQueryResultMapper.map(usageQueryResult)).build(); } catch (RemoteInvocationException | URISyntaxException e) { throw new RuntimeException(String.format("Failed to load Usage Stats for resource %s", key.getResource()), e); } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index d72a73a9887fc6..218deef3ab3b85 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -84,6 +84,11 @@ type Query { """ listPolicies(input: ListPoliciesInput!): ListPoliciesResult + """ + Get all granted privileges for the given actor and resource + """ + getGrantedPrivileges(input: GetGrantedPrivilegesInput!): Privileges + """ Incubating: Fetch an ML Model by primary key (urn) """ @@ -4755,18 +4760,18 @@ type Assertion implements EntityWithRelationships & Entity { The primary key of the Assertion """ urn: String! - + """ The standard Entity Type """ type: EntityType! - + """ Standardized platform urn where the assertion is evaluated """ platform: DataPlatform! - """ + """ Details about assertion """ info: AssertionInfo @@ -4774,7 +4779,7 @@ type Assertion implements EntityWithRelationships & Entity { """ Lifecycle events detailing individual runs of this assertion. If startTimeMillis & endTimeMillis are not provided, the most recent events will be returned. - """ + """ runEvents(status: AssertionRunStatus, startTimeMillis: Long, endTimeMillis: Long, limit: Int): AssertionRunEventsResult """ @@ -4952,7 +4957,7 @@ type AssertionResult { URL where full results are available """ externalUrl: String - + """ Native results / properties of evaluation """ @@ -4975,7 +4980,7 @@ type BatchSpec { Any limit to the number of rows in the batch, if applied """ limit: Int - + """ Custom properties of the Batch """ @@ -5017,7 +5022,7 @@ enum AssertionStdAggregation { IDENTITY """ - Assertion is applied on column mean + Assertion is applied on column mean """ MEAN @@ -5124,7 +5129,7 @@ enum AssertionStdOperator { """ Value being asserted is not null """ - NOT_NULL + NOT_NULL """ Value being asserted contains value @@ -5148,7 +5153,7 @@ enum AssertionStdOperator { """ Other - """ + """ _NATIVE_ } @@ -5727,7 +5732,7 @@ type ResourceFilter { """ The type of the resource the policy should apply to Not required because in the future we want to support filtering by type OR by domain """ - type: String! + type: String """ A list of specific resource urns to apply the filter to @@ -5737,7 +5742,12 @@ type ResourceFilter { """ Whether of not to apply the filter to all resources of the type """ - allResources: Boolean! + allResources: Boolean + + """ + Whether of not to apply the filter to all resources of the type + """ + filter: PolicyMatchFilter } """ @@ -5781,6 +5791,62 @@ type ActorFilter { resolvedGroups: [CorpGroup!] } +""" +Filter object that encodes a complex filter logic with OR + AND +""" +type PolicyMatchFilter { + """ + List of criteria to apply + """ + criteria: [PolicyMatchCriterion!] +} + +""" +Criterion to define relationship between field and values +""" +type PolicyMatchCriterion { + """ + The name of the field that the criterion refers to + e.g. entity_type, entity_urn, domain + """ + field: String! + + """ + Values. Matches criterion if any one of the values matches condition (OR-relationship) + """ + values: [PolicyMatchCriterionValue!]! + + """ + The name of the field that the criterion refers to + """ + condition: PolicyMatchCondition! +} + +""" +Value in PolicyMatchCriterion with hydrated entity if value is urn +""" +type PolicyMatchCriterionValue { + """ + The value of the field to match + """ + value: String! + + """ + Hydrated entities of the above values. Only set if the value is an urn + """ + entity: Entity +} + +""" +Match condition +""" +enum PolicyMatchCondition { + """ + Whether the field matches the value + """ + EQUALS +} + """ Input required when creating or updating an Access Policies Determines which resources the Policy applies to """ @@ -5789,7 +5855,7 @@ input ResourceFilterInput { The type of the resource the policy should apply to Not required because in the future we want to support filtering by type OR by domain """ - type: String! + type: String """ A list of specific resource urns to apply the filter to @@ -5799,7 +5865,12 @@ input ResourceFilterInput { """ Whether of not to apply the filter to all resources of the type """ - allResources: Boolean! + allResources: Boolean + + """ + Whether of not to apply the filter to all resources of the type + """ + filter: PolicyMatchFilterInput } """ @@ -5833,6 +5904,37 @@ input ActorFilterInput { allGroups: Boolean! } +""" +Filter object that encodes a complex filter logic with OR + AND +""" +input PolicyMatchFilterInput { + """ + List of criteria to apply + """ + criteria: [PolicyMatchCriterionInput!] +} + +""" +Criterion to define relationship between field and values +""" +input PolicyMatchCriterionInput { + """ + The name of the field that the criterion refers to + e.g. entity_type, entity_urn, domain + """ + field: String! + + """ + Values. Matches criterion if any one of the values matches condition (OR-relationship) + """ + values: [String!]! + + """ + The name of the field that the criterion refers to + """ + condition: PolicyMatchCondition! +} + """ Input required when listing DataHub Access Policies """ @@ -5873,6 +5975,46 @@ type ListPoliciesResult { policies: [Policy!]! } +""" +Input for getting granted privileges +""" +input GetGrantedPrivilegesInput { + """ + Urn of the actor + """ + actorUrn: String! + + """ + Spec to identify resource. If empty, gets privileges granted to the actor + """ + resourceSpec: ResourceSpec +} + +""" +Spec to identify resource +""" +input ResourceSpec { + """ + Resource type + """ + resourceType: EntityType! + + """ + Resource urn + """ + resourceUrn: String! +} + +""" +Object that encodes the privileges the actor has for a given resource +""" +type Privileges { + """ + Granted Privileges + """ + privileges: [String!]! +} + """ Input required when listing DataHub Users """ diff --git a/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx b/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx new file mode 100644 index 00000000000000..5312f4e09d3ab2 --- /dev/null +++ b/datahub-web-react/src/app/authorization/UnauthorizedPage.tsx @@ -0,0 +1,11 @@ +import { Result } from 'antd'; +import React from 'react'; +import { SearchablePage } from '../search/SearchablePage'; + +export const UnauthorizedPage = () => { + return ( + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/EntityPage.tsx b/datahub-web-react/src/app/entity/EntityPage.tsx index bfc37db3a29dfd..98c54878b88a87 100644 --- a/datahub-web-react/src/app/entity/EntityPage.tsx +++ b/datahub-web-react/src/app/entity/EntityPage.tsx @@ -1,4 +1,5 @@ import React, { useEffect } from 'react'; +import { Alert } from 'antd'; import { useParams } from 'react-router-dom'; import { EntityType } from '../../types.generated'; import { BrowsableEntityPage } from '../browse/BrowsableEntityPage'; @@ -8,6 +9,10 @@ import { SearchablePage } from '../search/SearchablePage'; import { useEntityRegistry } from '../useEntityRegistry'; import analytics, { EventType } from '../analytics'; import { decodeUrn } from './shared/utils'; +import { useGetAuthenticatedUserUrn } from '../useGetAuthenticatedUser'; +import { useGetGrantedPrivilegesQuery } from '../../graphql/policy.generated'; +import { Message } from '../shared/Message'; +import { UnauthorizedPage } from '../authorization/UnauthorizedPage'; interface RouteParams { urn: string; @@ -24,10 +29,22 @@ export const EntityPage = ({ entityType }: Props) => { const { urn: encodedUrn } = useParams(); const urn = decodeUrn(encodedUrn); const entityRegistry = useEntityRegistry(); - const isBrowsable = entityRegistry.getEntity(entityType).isBrowseEnabled(); - const isLineageSupported = entityRegistry.getEntity(entityType).isLineageEnabled(); + const entity = entityRegistry.getEntity(entityType); + const isBrowsable = entity.isBrowseEnabled(); + const isLineageSupported = entity.isLineageEnabled(); const ContainerPage = isBrowsable || isLineageSupported ? BrowsableEntityPage : SearchablePage; const isLineageMode = useIsLineageMode(); + const authenticatedUserUrn = useGetAuthenticatedUserUrn(); + const { loading, error, data } = useGetGrantedPrivilegesQuery({ + variables: { + input: { + actorUrn: authenticatedUserUrn, + resourceSpec: { resourceType: entityType, resourceUrn: urn }, + }, + }, + }); + const privileges = data?.getGrantedPrivileges?.privileges || []; + useEffect(() => { analytics.event({ type: EventType.EntityViewEvent, @@ -36,26 +53,35 @@ export const EntityPage = ({ entityType }: Props) => { }); }, [entityType, urn]); - // show new page for datasets - if ( + const canViewEntityPage = privileges.find((privilege) => privilege === 'VIEW_ENTITY_PAGE'); + const showNewPage = entityType === EntityType.Dataset || entityType === EntityType.Dashboard || entityType === EntityType.Chart || entityType === EntityType.DataFlow || entityType === EntityType.DataJob || - entityType === EntityType.GlossaryTerm - ) { - return {entityRegistry.renderProfile(entityType, urn)}; - } + entityType === EntityType.GlossaryTerm; - // show legacy page for other entities return ( - - {isLineageMode && isLineageSupported ? ( - - ) : ( - entityRegistry.renderProfile(entityType, urn) - )} - + <> + {loading && } + {error && } + {data && !canViewEntityPage && } + {canViewEntityPage && + ((showNewPage && {entityRegistry.renderProfile(entityType, urn)}) || ( + + {isLineageMode && isLineageSupported ? ( + + ) : ( + entityRegistry.renderProfile(entityType, urn) + )} + + ))} + ); }; diff --git a/datahub-web-react/src/app/policy/PoliciesPage.tsx b/datahub-web-react/src/app/policy/PoliciesPage.tsx index 3b6b67a980622c..21327ef0ee4b29 100644 --- a/datahub-web-react/src/app/policy/PoliciesPage.tsx +++ b/datahub-web-react/src/app/policy/PoliciesPage.tsx @@ -5,7 +5,17 @@ import { DeleteOutlined, PlusOutlined } from '@ant-design/icons'; import { SearchablePage } from '../search/SearchablePage'; import PolicyBuilderModal from './PolicyBuilderModal'; -import { Policy, PolicyUpdateInput, PolicyState, PolicyType, Maybe } from '../../types.generated'; +import { + Policy, + PolicyUpdateInput, + PolicyState, + PolicyType, + Maybe, + ResourceFilterInput, + PolicyMatchFilter, + PolicyMatchFilterInput, + PolicyMatchCriterionInput, +} from '../../types.generated'; import { useAppConfig } from '../useAppConfig'; import PolicyDetailsModal from './PolicyDetailsModal'; import { @@ -80,6 +90,18 @@ type PrivilegeOptionType = { name?: Maybe; }; +const toFilterInput = (filter: PolicyMatchFilter): PolicyMatchFilterInput => { + return { + criteria: filter.criteria?.map((criterion): PolicyMatchCriterionInput => { + return { + field: criterion.field, + values: criterion.values.map((criterionValue) => criterionValue.value), + condition: criterion.condition, + }; + }), + }; +}; + const toPolicyInput = (policy: Omit): PolicyUpdateInput => { let policyInput: PolicyUpdateInput = { type: policy.type, @@ -96,14 +118,18 @@ const toPolicyInput = (policy: Omit): PolicyUpdateInput => { }, }; if (policy.resources !== null && policy.resources !== undefined) { + let resourceFilter: ResourceFilterInput = { + type: policy.resources.type, + resources: policy.resources.resources, + allResources: policy.resources.allResources, + }; + if (policy.resources.filter) { + resourceFilter = { ...resourceFilter, filter: toFilterInput(policy.resources.filter) }; + } // Add the resource filters. policyInput = { ...policyInput, - resources: { - type: policy.resources.type, - resources: policy.resources.resources, - allResources: policy.resources.allResources, - }, + resources: resourceFilter, }; } return policyInput; @@ -187,17 +213,15 @@ export const PoliciesPage = () => { return { type: platformPrivilege.type, name: platformPrivilege.displayName }; }); } else { - const privilegeData = resourcePrivileges.filter( - (resourcePrivilege) => resourcePrivilege.resourceType === policy?.resources?.type, + const allResourcePriviliges = resourcePrivileges.find( + (resourcePrivilege) => resourcePrivilege.resourceType === 'all', ); privileges = - (privilegeData.length > 0 && - privilegeData[0]?.privileges - .filter((resourcePrivilege) => policy.privileges.includes(resourcePrivilege.type)) - .map((b) => { - return { type: b.type, name: b.displayName }; - })) || - []; + allResourcePriviliges?.privileges + .filter((resourcePrivilege) => policy.privileges.includes(resourcePrivilege.type)) + .map((b) => { + return { type: b.type, name: b.displayName }; + }) || []; } return privileges; }; @@ -379,7 +403,7 @@ export const PoliciesPage = () => { return ( - {policiesLoading && } + {policiesLoading && } {policiesError && message.error('Failed to load policies :(')} {updateError && message.error('Failed to update the Policy :(')} diff --git a/datahub-web-react/src/app/policy/PolicyBuilderModal.tsx b/datahub-web-react/src/app/policy/PolicyBuilderModal.tsx index ae8a950163b9fa..6ef9ace66ca394 100644 --- a/datahub-web-react/src/app/policy/PolicyBuilderModal.tsx +++ b/datahub-web-react/src/app/policy/PolicyBuilderModal.tsx @@ -91,7 +91,9 @@ export default function PolicyBuilderModal({ policy, setPolicy, visible, onClose setPolicy({ ...policy, resources })} + setResources={(resources: ResourceFilter) => { + setPolicy({ ...policy, resources }); + }} privileges={policy.privileges} setPrivileges={(privileges: string[]) => setPolicy({ ...policy, privileges })} /> diff --git a/datahub-web-react/src/app/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/policy/PolicyDetailsModal.tsx index 880b6418de839b..a6704ac3c1e218 100644 --- a/datahub-web-react/src/app/policy/PolicyDetailsModal.tsx +++ b/datahub-web-react/src/app/policy/PolicyDetailsModal.tsx @@ -4,7 +4,7 @@ import styled from 'styled-components'; import { useEntityRegistry } from '../useEntityRegistry'; import { Maybe, Policy, PolicyState, PolicyType } from '../../types.generated'; import { useAppConfig } from '../useAppConfig'; -import { mapResourceTypeToDisplayName } from './policyUtils'; +import { convertLegacyResourceFilter, getFieldValues, mapResourceTypeToDisplayName } from './policyUtils'; import AvatarsGroup from './AvatarsGroup'; type PrivilegeOptionType = { @@ -65,10 +65,22 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege const isActive = policy?.state === PolicyState.Active; const isMetadataPolicy = policy?.type === PolicyType.Metadata; + const resources = convertLegacyResourceFilter(policy?.resources); + const resourceTypes = getFieldValues(resources?.filter, 'RESOURCE_TYPE') || []; + const resourceEntities = getFieldValues(resources?.filter, 'RESOURCE_URN') || []; + const domains = getFieldValues(resources?.filter, 'DOMAIN') || []; + const { config: { policiesConfig }, } = useAppConfig(); + const getDisplayName = (entity) => { + if (!entity) { + return null; + } + return entityRegistry.getDisplayName(entity.type, entity); + }; + const actionButtons = ( @@ -98,26 +110,50 @@ export default function PolicyDetailsModal({ policy, visible, onClose, privilege
Asset Type - - {mapResourceTypeToDisplayName( - policy?.resources?.type || '', - policiesConfig?.resourcePrivileges || [], - )} - + {(resourceTypes?.length && + resourceTypes.map((value, key) => { + return ( + // eslint-disable-next-line react/no-array-index-key + + + {mapResourceTypeToDisplayName( + value.value, + policiesConfig?.resourcePrivileges || [], + )} + + + ); + })) || All}
Assets - {policy?.resources?.resources?.map((urn, key) => { - // TODO: Wrap in a link for entities. - return ( - // eslint-disable-next-line react/no-array-index-key - - {urn} - - ); - })} - {policy?.resources?.allResources && All} + {(resourceEntities?.length && + resourceEntities.map((value, key) => { + return ( + // eslint-disable-next-line react/no-array-index-key + + + {getDisplayName(value.entity) || value.value} + + + ); + })) || All} +
+
+ Domains + + {(domains?.length && + domains.map((value, key) => { + return ( + // eslint-disable-next-line react/no-array-index-key + + + {getDisplayName(value.entity) || value.value} + + + ); + })) || All}
)} diff --git a/datahub-web-react/src/app/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/policy/PolicyPrivilegeForm.tsx index 978cfd4347336e..7ccab4d7281e49 100644 --- a/datahub-web-react/src/app/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/policy/PolicyPrivilegeForm.tsx @@ -1,17 +1,21 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import { Form, Select, Tag, Typography } from 'antd'; -import { Link } from 'react-router-dom'; import styled from 'styled-components'; import { useEntityRegistry } from '../useEntityRegistry'; import { useAppConfig } from '../useAppConfig'; -import { useGetSearchResultsLazyQuery } from '../../graphql/search.generated'; -import { ResourceFilter, PolicyType } from '../../types.generated'; +import { useGetSearchResultsForMultipleLazyQuery, useGetSearchResultsLazyQuery } from '../../graphql/search.generated'; +import { ResourceFilter, PolicyType, EntityType } from '../../types.generated'; import { + convertLegacyResourceFilter, + createCriterionValue, + createCriterionValueWithEntity, EMPTY_POLICY, + getFieldValues, mapResourceTypeToDisplayName, mapResourceTypeToEntityType, mapResourceTypeToPrivileges, + setFieldValues, } from './policyUtils'; type Props = { @@ -28,13 +32,6 @@ const PrivilegesForm = styled(Form)` margin-bottom: 40px; `; -const SearchResultContainer = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - padding: 12px; -`; - /** * Component used to construct the "privileges" and "resources" portion of a DataHub * access Policy. @@ -53,38 +50,57 @@ export default function PolicyPrivilegeForm({ config: { policiesConfig }, } = useAppConfig(); - // Search for resources - const [search, { data: searchData }] = useGetSearchResultsLazyQuery(); - const resourceSearchResults = searchData?.search?.searchResults; + const resources: ResourceFilter = convertLegacyResourceFilter(maybeResources) || EMPTY_POLICY.resources; + const resourceTypes = getFieldValues(resources.filter, 'RESOURCE_TYPE') || []; + const resourceEntities = getFieldValues(resources.filter, 'RESOURCE_URN') || []; + + const getDisplayName = (entity) => { + if (!entity) { + return null; + } + return entityRegistry.getDisplayName(entity.type, entity); + }; - const resources = maybeResources || EMPTY_POLICY.resources; + const resourceUrnToDisplayName = new Map(); + resourceEntities.forEach((resourceEntity) => { + resourceUrnToDisplayName[resourceEntity.value] = getDisplayName(resourceEntity.entity); + }); + // Search for resources + const [searchResources, { data: resourcesSearchData }] = useGetSearchResultsForMultipleLazyQuery(); + const resourceSearchResults = resourcesSearchData?.searchAcrossEntities?.searchResults; - // Whether to show the "resource type" input. - const showResourceTypeInput = policyType !== PolicyType.Platform; + // Same for domains + const domains = getFieldValues(resources.filter, 'DOMAIN') || []; + const domainUrnToDisplayName = new Map(); + domains.forEach((domainEntity) => { + domainUrnToDisplayName[domainEntity.value] = getDisplayName(domainEntity.entity); + }); + // Search for domains + const [searchDomains, { data: domainsSearchData }] = useGetSearchResultsLazyQuery(); + const domainSearchResults = domainsSearchData?.search?.searchResults; - // Whether to show the "resource" input. - const showResourceInput = showResourceTypeInput && resources.type !== ''; + // Whether to show the resource filter inputs including "resource type", "resource", and "domain" + const showResourceFilterInput = policyType !== PolicyType.Platform; - // Whether to show the "privileges" input. - const showPrivilegesInput = - policyType === PolicyType.Platform || - resources.allResources || - (resources.resources && resources.resources?.length > 0); + // Current Select dropdown values + const resourceTypeSelectValue = resourceTypes.map((criterionValue) => criterionValue.value); + const resourceSelectValue = resourceEntities.map((criterionValue) => criterionValue.value); + const domainSelectValue = getFieldValues(resources.filter, 'DOMAIN').map((criterionValue) => criterionValue.value); + const privilegesSelectValue = privileges; // Construct privilege options for dropdown const platformPrivileges = policiesConfig?.platformPrivileges || []; - const resourcePrivileges = policiesConfig?.resourcePrivileges || []; - const privilegeOptions = - policyType === PolicyType.Platform - ? platformPrivileges - : mapResourceTypeToPrivileges(resources.type, resourcePrivileges); + const resourcePrivileges = useMemo(() => policiesConfig?.resourcePrivileges || [], [policiesConfig]); + const resourcePrivilegesForType = useMemo( + () => mapResourceTypeToPrivileges(resourceTypeSelectValue, resourcePrivileges), + [resourceTypeSelectValue, resourcePrivileges], + ); + const privilegeOptions = policyType === PolicyType.Platform ? platformPrivileges : resourcePrivilegesForType; - // Current Select dropdown values - const resourceTypeSelectValue = resources.type; - const resourceSelectValue = resources.allResources ? ['All'] : resources.resources || []; - const privilegesSelectValue = privileges; + const getEntityFromSearchResults = (searchResults, urn) => + searchResults?.map((result) => result.entity).find((entity) => entity.urn === urn); - // When a privilege is selected, add its type to the privileges list + // When a privilege is selected, add its type to the privileges list test const onSelectPrivilege = (privilege: string) => { if (privilege === 'All') { setPrivileges(privilegeOptions.map((priv) => priv.type) as never[]); @@ -106,82 +122,151 @@ export default function PolicyPrivilegeForm({ }; // When a resource is selected, add its urn to the list of resources - const onSelectResource = (resource: string) => { - if (resource === 'All') { - setResources({ - ...resources, - allResources: true, - }); - } else { - const newAssets = [...(resources.resources || []), resource as string]; - setResources({ - ...resources, - resources: newAssets, - }); - } + const onSelectResourceType = (selectedResourceType: string) => { + const filter = resources.filter || { + criteria: [], + }; + setResources({ + ...resources, + filter: setFieldValues(filter, 'RESOURCE_TYPE', [ + ...resourceTypes, + createCriterionValue(selectedResourceType), + ]), + }); + }; + + const onDeselectResourceType = (deselectedResourceType: string) => { + const filter = resources.filter || { + criteria: [], + }; + setResources({ + ...resources, + filter: setFieldValues( + filter, + 'RESOURCE_TYPE', + resourceTypes?.filter((criterionValue) => criterionValue.value !== deselectedResourceType), + ), + }); + }; + + // When a resource is selected, add its urn to the list of resources + const onSelectResource = (resource) => { + const filter = resources.filter || { + criteria: [], + }; + setResources({ + ...resources, + filter: setFieldValues(filter, 'RESOURCE_URN', [ + ...resourceEntities, + createCriterionValueWithEntity( + resource, + getEntityFromSearchResults(resourceSearchResults, resource) || null, + ), + ]), + }); }; // When a resource is deselected, remove its urn from the list of resources - const onDeselectResource = (resource: string) => { - if (resource === 'All') { - setResources({ - ...resources, - allResources: false, - }); - } else { - const newAssets = resources.resources?.filter((urn) => urn !== resource); - setResources({ - ...resources, - resources: newAssets, - }); - } + const onDeselectResource = (resource) => { + const filter = resources.filter || { + criteria: [], + }; + setResources({ + ...resources, + filter: setFieldValues( + filter, + 'RESOURCE_URN', + resourceEntities?.filter((criterionValue) => criterionValue.value !== resource), + ), + }); + }; + + // When a domain is selected, add its urn to the list of domains + const onSelectDomain = (domain) => { + const filter = resources.filter || { + criteria: [], + }; + const updatedFilter = setFieldValues(filter, 'DOMAIN', [ + ...domains, + createCriterionValueWithEntity(domain, getEntityFromSearchResults(domainSearchResults, domain) || null), + ]); + setResources({ + ...resources, + filter: updatedFilter, + }); + }; + + // When a domain is deselected, remove its urn from the list of domains + const onDeselectDomain = (domain) => { + const filter = resources.filter || { + criteria: [], + }; + setResources({ + ...resources, + filter: setFieldValues( + filter, + 'DOMAIN', + domains?.filter((criterionValue) => criterionValue.value !== domain), + ), + }); }; // Handle resource search, if the resource type has an associated EntityType mapping. - const handleSearch = (text: string) => { - const maybeEntityType = mapResourceTypeToEntityType(resources.type, resourcePrivileges); - if (maybeEntityType) { - if (text.length > 2) { - search({ - variables: { - input: { - type: maybeEntityType, - query: text, - start: 0, - count: 10, - }, + const handleResourceSearch = (text: string) => { + const entityTypes = resourceTypeSelectValue + .map((resourceType) => mapResourceTypeToEntityType(resourceType, resourcePrivileges)) + .filter((entityType): entityType is EntityType => !!entityType); + if (text.length > 2) { + searchResources({ + variables: { + input: { + types: entityTypes, + query: text, + start: 0, + count: 10, }, - }); - } + }, + }); } }; - const renderSearchResult = (result) => { - return ( - - {entityRegistry.getDisplayName(result.entity.type, result.entity)} - `/${entityRegistry.getPathName(result.entity.type)}/${result.entity.urn}`} - > - View - - - ); + // Handle domain search, if the domain type has an associated EntityType mapping. + const handleDomainSearch = (text: string) => { + if (text.length > 2) { + searchDomains({ + variables: { + input: { + type: EntityType.Domain, + query: text, + start: 0, + count: 10, + }, + }, + }); + } }; - const selectedResourceDisplayName = - showResourceInput && mapResourceTypeToDisplayName(resources.type, resourcePrivileges); - return ( - {showResourceTypeInput && ( + {showResourceFilterInput && ( Resource Type} labelAlign="right"> - Select the specific type of resource this policy should apply to. + Select the types of resources this policy should apply to. If none is selected, policy is + applied to all types of resources. - ( + + {mapResourceTypeToDisplayName(tagProps.value.toString(), resourcePrivileges)} + + )} + > {resourcePrivileges.map((resPrivs) => { return ( @@ -192,54 +277,77 @@ export default function PolicyPrivilegeForm({ )} - {showResourceInput && ( + {showResourceFilterInput && ( Resource}> - Search for specific resources the policy should apply to, or select All to apply the - policy to all resources of the given type. + Search for specific resources the policy should apply to. If none is selected, policy is + applied to all resources of the given type. )} - {showPrivilegesInput && ( - Privileges}> - Select a set of privileges to permit. + {showResourceFilterInput && ( + Domain}> + + Search for domains the policy should apply to. If none is selected, policy is applied to{' '} + all resources in all domains. + )} + Privileges}> + Select a set of privileges to permit. + + ); } diff --git a/datahub-web-react/src/app/policy/policyUtils.ts b/datahub-web-react/src/app/policy/policyUtils.ts index d4a71eb979692f..cfbc43e769268b 100644 --- a/datahub-web-react/src/app/policy/policyUtils.ts +++ b/datahub-web-react/src/app/policy/policyUtils.ts @@ -1,4 +1,15 @@ -import { PolicyState, PolicyType, ResourcePrivileges } from '../../types.generated'; +import { Maybe } from 'graphql/jsutils/Maybe'; +import { + PolicyMatchCondition, + PolicyMatchCriterion, + PolicyMatchCriterionValue, + PolicyMatchFilter, + PolicyState, + PolicyType, + Privilege, + ResourceFilter, + ResourcePrivileges, +} from '../../types.generated'; export const EMPTY_POLICY = { type: PolicyType.Metadata, @@ -7,9 +18,9 @@ export const EMPTY_POLICY = { state: PolicyState.Active, privileges: new Array(), resources: { - type: '', - resources: new Array(), - allResources: false, + filter: { + criteria: [], + }, }, actors: { users: new Array(), @@ -43,10 +54,72 @@ export const mapResourceTypeToDisplayName = ( return ''; }; -export const mapResourceTypeToPrivileges = (resourceType: string, resourcePrivilegesArr: Array) => { - const resourcePrivileges = resourcePrivilegesArr.filter((privs) => privs.resourceType === resourceType); - if (resourcePrivileges.length === 1) { - return resourcePrivileges[0].privileges; +export const mapResourceTypeToPrivileges = ( + resourceTypes: string[], + resourcePrivilegesArr: Array, +) => { + if (resourceTypes.length === 0) { + return resourcePrivilegesArr.find((privs) => privs.resourceType === 'all')?.privileges || []; + } + // If input resource types are empty, consider all privileges + const resourcePrivileges = resourcePrivilegesArr.filter((privs) => resourceTypes.includes(privs.resourceType)); + if (resourcePrivileges.length > 0) { + const finalPrivileges: Privilege[] = []; + const uniquePrivileges = new Set(); + resourcePrivileges.forEach((resourcePrivilege) => { + resourcePrivilege.privileges.forEach((privilege) => { + if (!uniquePrivileges.has(privilege.type)) { + uniquePrivileges.add(privilege.type); + finalPrivileges.push(privilege); + } + }); + }); + return finalPrivileges; } return []; }; + +export const createCriterion = ( + resourceFieldType: string, + fieldValues: Array, +): PolicyMatchCriterion => ({ + field: resourceFieldType, + values: fieldValues, + condition: PolicyMatchCondition.Equals, +}); + +export const createCriterionValue = (value: string): PolicyMatchCriterionValue => ({ value }); + +export const createCriterionValueWithEntity = (value: string, entity): PolicyMatchCriterionValue => ({ value, entity }); + +export const convertLegacyResourceFilter = (resourceFilter: Maybe | undefined) => { + // If empty or filter is set, resource filter is valid so return itself + if (!resourceFilter || resourceFilter.filter) { + return resourceFilter; + } + const criteria = new Array(); + if (resourceFilter.type) { + criteria.push(createCriterion('RESOURCE_TYPE', [createCriterionValue(resourceFilter.type)])); + } + if (resourceFilter.resources && resourceFilter.resources.length > 0) { + criteria.push(createCriterion('RESOURCE_URN', resourceFilter.resources.map(createCriterionValue))); + } + return { + filter: { + criteria, + }, + }; +}; + +export const getFieldValues = (filter: Maybe | undefined, resourceFieldType: string) => { + return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values || []; +}; + +export const setFieldValues = ( + filter: PolicyMatchFilter, + resourceFieldType: string, + fieldValues: Array, +): PolicyMatchFilter => { + const restCriteria = filter.criteria?.filter((criterion) => criterion.field !== resourceFieldType) || []; + return { ...filter, criteria: [...restCriteria, createCriterion(resourceFieldType, fieldValues)] }; +}; diff --git a/datahub-web-react/src/graphql/policy.graphql b/datahub-web-react/src/graphql/policy.graphql index a9b2f9a207c301..c3643ac677b273 100644 --- a/datahub-web-react/src/graphql/policy.graphql +++ b/datahub-web-react/src/graphql/policy.graphql @@ -13,6 +13,18 @@ query listPolicies($input: ListPoliciesInput!) { type allResources resources + filter { + criteria { + field + values { + value + entity { + ...searchResultFields + } + } + condition + } + } } privileges actors { @@ -61,6 +73,12 @@ query listPolicies($input: ListPoliciesInput!) { } } +query getGrantedPrivileges($input: GetGrantedPrivilegesInput!) { + getGrantedPrivileges(input: $input) { + privileges + } +} + mutation createPolicy($input: PolicyUpdateInput!) { createPolicy(input: $input) } diff --git a/docs/policies.md b/docs/policies.md index 52b062e39a5c88..584b54c4c2ff4a 100644 --- a/docs/policies.md +++ b/docs/policies.md @@ -53,7 +53,72 @@ A Metadata Policy can be broken down into 3 parts: 2. **Privileges**: The 'what'. What actions are being permitted by a policy, e.g. "Add Tags". 3. **Resources**: The 'which'. Resources that the policy applies to, e.g. "All Datasets". -> Today, the set of privileges supported includes only *write* privileges. That is, there are no read restrictions implemented yet. +#### Actors + +We currently support 3 ways to define the set of actors the policy applies to: a) list of users b) list of groups, and +c) owners of the entity. You also have the option to apply the policy to all users. + +#### Privileges + +Check out the list of +privileges [here](https://github.com/datahub-project/datahub/blob/master/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java) +. Note, the privileges are semantic by nature, and does not tie in 1-to-1 with the aspect model. + +All edits on the UI are covered by a privilege, to make sure we have the ability to restrict write access. + + + +We currently support the following read privileges + +| Privilege | Description | +|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| VIEW_ENTITY_PAGE | Allow actor to access the entity page for the resource in the UI. If not granted, it will redirect them to an unauthorized page. | +| VIEW_DATASET_USAGE | Allow actor to access usage metadata about a dataset both in the UI and in the GraphQL API. This includes example queries, number of queries, etc. | +| VIEW_DATASET_PROFILE | Allow actor to access a dataset's profile both in the UI and in the GraphQL API. This includes snapshot statistics like #rows, #columns, null percentage per field, etc. | + +#### Resources + +Resource filter defines the set of resources that the policy applies to is defined using a list of criteria. Each +criterion defines a field type (like resource_type, resource_urn, domain), a list of field values to compare, and a +condition (like EQUALS). It essentially checks whether the field of a certain resource matches any of the input values. +Note, that if there are no criteria or resource is not set, policy is applied to ALL resources. + +For example, the following resource filter will apply the policy to datasets, charts, and dashboards under domain 1. + +```json +{ + "resource": { + "criteria": [ + { + "field": "resource_type", + "values": [ + "dataset", + "chart", + "dashboard" + ], + "condition": "EQUALS" + }, + { + "field": "domain", + "values": [ + "urn:li:domain:domain1" + ], + "condition": "EQUALS" + } + ] + } +} +``` + +Supported fields are as follows + +| Field Type | Description | Example | +|---------------|------------------------|-------------------------| +| resource_type | Type of the resource | dataset, chart, dataJob | +| resource_urn | Urn of the resource | urn:li:dataset:... | +| domain | Domain of the resource | urn:li:domain:domainX | ## Managing Policies @@ -102,7 +167,6 @@ The DataHub team is hard at work trying to improve the Policies feature. We are Under consideration -- Ability to define Metadata Policies against multiple resources scoped to a particular "Domains" - Ability to define Metadata Policies against multiple reosurces scoped to particular "Containers" (e.g. A "schema", "database", or "collection") ## Feedback / Questions / Concerns diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java index 2f45deb1d63cbc..8dabf4a7e89c44 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityService.java @@ -54,8 +54,9 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.utils.PegasusUtils.*; +import static com.linkedin.metadata.Constants.ASPECT_LATEST_VERSION; +import static com.linkedin.metadata.utils.PegasusUtils.getDataTemplateClassFromSchema; +import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; /** @@ -142,6 +143,23 @@ public abstract Map> getLatestAspects( @Nullable public abstract RecordTemplate getAspect(@Nonnull final Urn urn, @Nonnull final String aspectName, long version); + /** + * Retrieves the latest aspects for the given urn as dynamic aspect objects + * (Without having to define union objects) + * + * @param entityName name of the entity to fetch + * @param urn urn of entity to fetch + * @param aspectNames set of aspects to fetch + * @return a map of {@link Urn} to {@link Entity} object + */ + @Nullable + public EntityResponse getEntityV2( + @Nonnull final String entityName, + @Nonnull final Urn urn, + @Nonnull final Set aspectNames) throws URISyntaxException { + return getEntitiesV2(entityName, Collections.singleton(urn), aspectNames).get(urn); + } + /** * Retrieves the latest aspects for the given set of urns as dynamic aspect objects * (Without having to define union objects) @@ -360,7 +378,7 @@ private RecordTemplate sendEventForUpdateAspectResult(@Nonnull final Urn urn, @N result.getNewSystemMetadata(), MetadataAuditOperation.UPDATE); produceMAETimer.stop(); } else { - log.debug("Skipped producing MetadataAuditEvent for ingested aspect {}, urn {}. Aspect has not changed.", + log.debug("Skipped producing MetadataAuditEvent for ingested aspect {}, urn {}. Aspect has not changed.", aspectName, urn); } return updatedValue; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index 2c28e5016e717a..9fcddc9cb978ce 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -349,14 +349,17 @@ public ListResult listVersions( @Nonnull public ListResult listUrns( + @Nonnull final String entityName, @Nonnull final String aspectName, final int start, final int pageSize) { validateConnection(); + final String urnPrefixMatcher = "urn:li:" + entityName + ":%"; final PagedList pagedList = _server.find(EbeanAspectV2.class) .select(EbeanAspectV2.KEY_ID) .where() + .like(EbeanAspectV2.URN_COLUMN, urnPrefixMatcher) .eq(EbeanAspectV2.ASPECT_COLUMN, aspectName) .eq(EbeanAspectV2.VERSION_COLUMN, ASPECT_LATEST_VERSION) .setFirstRow(start) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java index d878147f3c75df..f27ffcea5ce0c2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanEntityService.java @@ -684,7 +684,7 @@ public ListUrnsResult listUrns(@Nonnull final String entityName, final int start // If a keyAspect exists, the entity exists. final String keyAspectName = getEntityRegistry().getEntitySpec(entityName).getKeyAspectSpec().getName(); - final ListResult keyAspectList = _entityDao.listUrns(keyAspectName, start, count); + final ListResult keyAspectList = _entityDao.listUrns(entityName, keyAspectName, start, count); final ListUrnsResult result = new ListUrnsResult(); result.setStart(start); diff --git a/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubResourceFilter.pdl b/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubResourceFilter.pdl index 52f57d3b094237..4767900868d2c1 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubResourceFilter.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/policy/DataHubResourceFilter.pdl @@ -12,15 +12,23 @@ record DataHubResourceFilter { * example 'dataset'. It is not strictly required because in the future we will want to support filtering a resource * by domain, as well. */ + @deprecated type: optional string /** * A specific set of resources to apply the policy to, e.g. asset urns */ + @deprecated resources: optional array[string] /** * Whether the policy should be applied to all assets matching the filter. */ - allResources: boolean = false + @deprecated + allResources: boolean = false + + /** + * Filter to apply privileges to + */ + filter: optional PolicyMatchFilter } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCondition.pdl b/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCondition.pdl new file mode 100644 index 00000000000000..0c51e7072ebd28 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCondition.pdl @@ -0,0 +1,11 @@ +namespace com.linkedin.policy + +/** + * The matching condition in a filter criterion + */ +enum PolicyMatchCondition { + /** + * Whether the field matches the value + */ + EQUALS +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCriterion.pdl b/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCriterion.pdl new file mode 100644 index 00000000000000..707a39697f3fd9 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchCriterion.pdl @@ -0,0 +1,23 @@ +namespace com.linkedin.policy + +/** + * A criterion for matching a field with given value + */ +record PolicyMatchCriterion { + + /** + * The name of the field that the criterion refers to + */ + field: string + + /** + * Values. Matches criterion if any one of the values matches condition (OR-relationship) + */ + values: array[string] + + /** + * The condition for the criterion + */ + condition: PolicyMatchCondition = "EQUALS" + +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchFilter.pdl b/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchFilter.pdl new file mode 100644 index 00000000000000..ed4262578fa1af --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/policy/PolicyMatchFilter.pdl @@ -0,0 +1,13 @@ +namespace com.linkedin.policy + +/** + * The filter for specifying the resource or actor to apply privileges to + */ +record PolicyMatchFilter { + + /** + * A list of criteria to apply conjunctively (so all criteria must pass) + */ + criteria: array[PolicyMatchCriterion] + +} \ No newline at end of file diff --git a/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java b/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java index c4bc3fe5eba66e..1acfc96c41f9e3 100644 --- a/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java +++ b/metadata-service/auth-api/src/main/java/com/datahub/authorization/AuthorizationRequest.java @@ -1,58 +1,12 @@ package com.datahub.authorization; -import java.util.Objects; import java.util.Optional; +import lombok.Value; -public class AuthorizationRequest { - - private final String _actorUrn; - private final String _privilege; - private final Optional _resourceSpec; - - public AuthorizationRequest( - final String actorUrn, // urn:li:corpuser:datahub, urn:li:corpGroup:engineering - final String privilege, - final Optional resourceSpec) { - _actorUrn = actorUrn; - _privilege = privilege; - _resourceSpec = resourceSpec; - } - - public String actorUrn() { - return _actorUrn; - } - - public String privilege() { - return _privilege; - } - - public Optional resourceSpec() { - return _resourceSpec; - } - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - - if (!(o instanceof AuthorizationRequest)) { - return false; - } - - AuthorizationRequest c = (AuthorizationRequest) o; - - return this._actorUrn.equals(c._actorUrn) - && this._privilege.equals(c._privilege) - && this._resourceSpec.equals(c._resourceSpec); - } - - @Override - public int hashCode() { - return Objects.hash( - this._actorUrn, - this._privilege, - this._resourceSpec - ); - } +@Value +public class AuthorizationRequest { + String actorUrn; + String privilege; + Optional resourceSpec; } diff --git a/metadata-service/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java b/metadata-service/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java index 79a2a9e0d4b412..c8bdc96362ad9c 100644 --- a/metadata-service/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java +++ b/metadata-service/auth-api/src/main/java/com/datahub/authorization/ResourceSpec.java @@ -1,25 +1,14 @@ package com.datahub.authorization; import javax.annotation.Nonnull; +import lombok.Value; -public class ResourceSpec { - - private final String _type; - private final String _resource; - public ResourceSpec( - @Nonnull final String type, - @Nonnull final String resource // urn:li:dataset:(123) - ) { - _type = type; - _resource = resource; - } - - public String getType() { - return _type; - } +@Value +public class ResourceSpec { + @Nonnull + String type; - public String getResource() { - return _resource; - } + @Nonnull + String resource; } \ No newline at end of file diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizationManager.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizationManager.java index 67831a2dd42360..54cd0d3fe9507a 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizationManager.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/AuthorizationManager.java @@ -5,10 +5,10 @@ import com.linkedin.common.Owner; import com.linkedin.common.Ownership; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.OwnershipClient; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.policy.DataHubPolicyInfo; @@ -26,7 +26,10 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.CORP_GROUP_ENTITY_NAME; +import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATAHUB_POLICY_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.POLICY_ENTITY_NAME; /** @@ -50,13 +53,15 @@ public class AuthorizationManager implements Authorizer { private final ScheduledExecutorService _refreshExecutorService = Executors.newScheduledThreadPool(1); private final PolicyRefreshRunnable _policyRefreshRunnable; + private final ResourceSpecResolver _resourceSpecResolver; private final PolicyEngine _policyEngine; private AuthorizationMode _mode; + public static final String ALL = "ALL"; + public AuthorizationManager( final Authentication systemAuthentication, final EntityClient entityClient, - final OwnershipClient ownershipClient, final int delayIntervalSeconds, final int refreshIntervalSeconds, final AuthorizationMode mode) { @@ -64,7 +69,8 @@ public AuthorizationManager( _policyRefreshRunnable = new PolicyRefreshRunnable(systemAuthentication, entityClient, _policyCache); _refreshExecutorService.scheduleAtFixedRate(_policyRefreshRunnable, delayIntervalSeconds, refreshIntervalSeconds, TimeUnit.SECONDS); _mode = mode; - _policyEngine = new PolicyEngine(systemAuthentication, entityClient, ownershipClient); + _resourceSpecResolver = new ResourceSpecResolver(systemAuthentication, entityClient); + _policyEngine = new PolicyEngine(systemAuthentication, entityClient); } public AuthorizationResult authorize(final AuthorizationRequest request) { @@ -74,12 +80,14 @@ public AuthorizationResult authorize(final AuthorizationRequest request) { return new AuthorizationResult(request, Optional.empty(), AuthorizationResult.Type.ALLOW); } + Optional resolvedResourceSpec = request.getResourceSpec().map(_resourceSpecResolver::resolve); + // 1. Fetch the policies relevant to the requested privilege. - final List policiesToEvaluate = _policyCache.getOrDefault(request.privilege(), new ArrayList<>()); + final List policiesToEvaluate = _policyCache.getOrDefault(request.getPrivilege(), new ArrayList<>()); // 2. Evaluate each policy. for (DataHubPolicyInfo policy : policiesToEvaluate) { - if (isRequestGranted(policy, request)) { + if (isRequestGranted(policy, request, resolvedResourceSpec)) { // Short circuit if policy has granted privileges to this actor. return new AuthorizationResult(request, Optional.of(policy), AuthorizationResult.Type.ALLOW); } @@ -87,14 +95,28 @@ public AuthorizationResult authorize(final AuthorizationRequest request) { return new AuthorizationResult(request, Optional.empty(), AuthorizationResult.Type.DENY); } + public List getGrantedPrivileges(final String actorUrn, final Optional resourceSpec) { + + // 1. Fetch all policies + final List policiesToEvaluate = _policyCache.getOrDefault(ALL, new ArrayList<>()); + + Optional resolvedResourceSpec = resourceSpec.map(_resourceSpecResolver::resolve); + + return _policyEngine.getGrantedPrivileges(policiesToEvaluate, UrnUtils.getUrn(actorUrn), resolvedResourceSpec); + } + /** * Retrieves the current list of actors authorized to for a particular privilege against * an optional resource */ - public AuthorizedActors authorizedActors(final String privilege, final Optional resourceSpec) throws RuntimeException { + public AuthorizedActors authorizedActors( + final String privilege, + final Optional resourceSpec) throws RuntimeException { // Step 1: Find policies granting the privilege. final List policiesToEvaluate = _policyCache.getOrDefault(privilege, new ArrayList<>()); + Optional resolvedResourceSpec = resourceSpec.map(_resourceSpecResolver::resolve); + final List authorizedUsers = new ArrayList<>(); final List authorizedGroups = new ArrayList<>(); boolean allUsers = false; @@ -107,7 +129,7 @@ public AuthorizedActors authorizedActors(final String privilege, final Optional< continue; } - final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resourceSpec); + final PolicyEngine.PolicyActors matchingActors = _policyEngine.getMatchingActors(policy, resolvedResourceSpec); // Step 3: For each matching policy, add actors that are authorized. authorizedUsers.addAll(matchingActors.getUsers()); @@ -146,21 +168,21 @@ public void setMode(final AuthorizationMode mode) { * the action is always authorized. */ private boolean isSystemRequest(final AuthorizationRequest request, final Authentication systemAuthentication) { - return systemAuthentication.getActor().toUrnStr().equals(request.actorUrn()); + return systemAuthentication.getActor().toUrnStr().equals(request.getActorUrn()); } /** * Returns true if a policy grants the requested privilege for a given actor and resource. */ - private boolean isRequestGranted(final DataHubPolicyInfo policy, final AuthorizationRequest request) { + private boolean isRequestGranted(final DataHubPolicyInfo policy, final AuthorizationRequest request, final Optional resourceSpec) { if (AuthorizationMode.ALLOW_ALL.equals(mode())) { return true; } final PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( policy, - request.actorUrn(), - request.privilege(), - request.resourceSpec() + request.getActorUrn(), + request.getPrivilege(), + resourceSpec ); return result.isGranted(); } @@ -247,6 +269,9 @@ private void addPolicyToCache(final Map> cache, existingPolicies.add(policy); cache.put(privilege, existingPolicies); } + List existingPolicies = cache.getOrDefault(ALL, new ArrayList<>()); + existingPolicies.add(policy); + cache.put(ALL, existingPolicies); } } diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FieldResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FieldResolver.java new file mode 100644 index 00000000000000..9318f5f8e7b96b --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FieldResolver.java @@ -0,0 +1,53 @@ +package com.datahub.authorization; + +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.function.Supplier; +import lombok.Builder; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Value; + + +/** + * Helper class for lazy resolution of fields + * Input resolveField function that is given as input will only be called when getFieldValuesFuture is called + */ +@RequiredArgsConstructor +public class FieldResolver { + private final Supplier> resolveField; + @Getter(lazy = true) + private final CompletableFuture fieldValuesFuture = resolveField.get(); + + private static final FieldValue EMPTY = new FieldValue(Collections.emptySet()); + + /** + * Helper function that returns FieldResolver for precomputed values + */ + public static FieldResolver getResolverFromValues(Set values) { + return new FieldResolver(() -> CompletableFuture.completedFuture(FieldValue.builder().values(values).build())); + } + + /** + * Helper function that returns FieldResolver given a fetchFieldValue function + */ + public static FieldResolver getResolverFromFunction(ResourceSpec resourceSpec, + Function fetchFieldValue) { + return new FieldResolver(() -> CompletableFuture.supplyAsync(() -> fetchFieldValue.apply(resourceSpec))); + } + + public static FieldValue emptyFieldValue() { + return EMPTY; + } + + /** + * Container for storing the field value, in case we need to extend this to have more types of field values + */ + @Value + @Builder + public static class FieldValue { + Set values; + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java new file mode 100644 index 00000000000000..70e2f093a02bb1 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/FilterUtils.java @@ -0,0 +1,68 @@ +package com.datahub.authorization; + +import com.datahub.authorization.fieldresolverprovider.ResourceFieldType; +import com.linkedin.data.template.StringArray; +import com.linkedin.policy.PolicyMatchCondition; +import com.linkedin.policy.PolicyMatchCriterion; +import com.linkedin.policy.PolicyMatchCriterionArray; +import com.linkedin.policy.PolicyMatchFilter; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + + +public class FilterUtils { + + public static final PolicyMatchFilter EMPTY_FILTER = + new PolicyMatchFilter().setCriteria(new PolicyMatchCriterionArray()); + + private FilterUtils() { + } + + /** + * Creates new PolicyMatchCriterion with field and value, using EQUAL PolicyMatchCondition. + */ + @Nonnull + public static PolicyMatchCriterion newCriterion(@Nonnull ResourceFieldType field, @Nonnull List values) { + return newCriterion(field, values, PolicyMatchCondition.EQUALS); + } + + /** + * Creates new PolicyMatchCriterion with field, value and PolicyMatchCondition. + */ + @Nonnull + public static PolicyMatchCriterion newCriterion(@Nonnull ResourceFieldType field, @Nonnull List values, + @Nonnull PolicyMatchCondition policyMatchCondition) { + return new PolicyMatchCriterion().setField(field.name()) + .setValues(new StringArray(values)) + .setCondition(policyMatchCondition); + } + + /** + * Creates new PolicyMatchFilter from a map of Criteria by removing null-valued Criteria and using EQUAL PolicyMatchCondition (default). + */ + @Nonnull + public static PolicyMatchFilter newFilter(@Nullable Map> params) { + if (params == null) { + return EMPTY_FILTER; + } + PolicyMatchCriterionArray criteria = params.entrySet() + .stream() + .filter(e -> Objects.nonNull(e.getValue())) + .map(e -> newCriterion(e.getKey(), e.getValue())) + .collect(Collectors.toCollection(PolicyMatchCriterionArray::new)); + return new PolicyMatchFilter().setCriteria(criteria); + } + + /** + * Creates new PolicyMatchFilter from a single PolicyMatchCriterion with EQUAL PolicyMatchCondition (default). + */ + @Nonnull + public static PolicyMatchFilter newFilter(@Nonnull ResourceFieldType field, @Nonnull List values) { + return newFilter(Collections.singletonMap(field, values)); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java index d2277012d33f4d..01a69b59fa9855 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/PolicyEngine.java @@ -1,18 +1,22 @@ package com.datahub.authorization; import com.datahub.authentication.Authentication; -import com.linkedin.common.Owner; -import com.linkedin.common.Ownership; +import com.datahub.authorization.fieldresolverprovider.ResourceFieldType; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.OwnershipClient; import com.linkedin.identity.GroupMembership; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; import com.linkedin.policy.DataHubResourceFilter; +import com.linkedin.policy.PolicyMatchCondition; +import com.linkedin.policy.PolicyMatchCriterion; +import com.linkedin.policy.PolicyMatchCriterionArray; +import com.linkedin.policy.PolicyMatchFilter; import com.linkedin.r2.RemoteInvocationException; import java.net.URISyntaxException; import java.util.ArrayList; @@ -24,31 +28,26 @@ import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.Constants.CORP_GROUP_ENTITY_NAME; +import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; +import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME; + @Slf4j +@RequiredArgsConstructor public class PolicyEngine { private final Authentication _systemAuthentication; private final EntityClient _entityClient; - private final OwnershipClient _ownershipClient; - - public PolicyEngine( - final Authentication systemAuthentication, - final EntityClient entityClient, - final OwnershipClient ownershipClient) { - _systemAuthentication = systemAuthentication; - _entityClient = entityClient; - _ownershipClient = ownershipClient; - } public PolicyEvaluationResult evaluatePolicy( final DataHubPolicyInfo policy, final String actorStr, final String privilege, - final Optional resource) { + final Optional resource) { try { // Currently Actor must be an urn. Consider whether this contract should be pushed up. final Urn actor = Urn.createFromString(actorStr); @@ -63,27 +62,17 @@ public PolicyEvaluationResult evaluatePolicy( final DataHubPolicyInfo policy, final Urn actor, final String privilege, - final Optional resource) { + final Optional resource) { final PolicyEvaluationContext context = new PolicyEvaluationContext(); - // If policy is inactive, simply return DENY. - if (PoliciesConfig.INACTIVE_POLICY_STATE.equals(policy.getState())) { - return PolicyEvaluationResult.DENIED; - } - // If the privilege is not in scope, deny the request. if (!isPrivilegeMatch(privilege, policy.getPrivileges(), context)) { return PolicyEvaluationResult.DENIED; } - // If the resource is not in scope, deny the request. - if (!isResourceMatch(policy.getType(), policy.getResources(), resource, context)) { - return PolicyEvaluationResult.DENIED; - } - - // If the actor does not match, deny the request. - if (!isActorMatch(actor, policy.getActors(), resource, context)) { + // If policy is not applicable, deny the request + if (!isPolicyApplicable(policy, actor, resource, context)) { return PolicyEvaluationResult.DENIED; } @@ -91,7 +80,9 @@ public PolicyEvaluationResult evaluatePolicy( return PolicyEvaluationResult.GRANTED; } - public PolicyActors getMatchingActors(final DataHubPolicyInfo policy, final Optional resource) { + public PolicyActors getMatchingActors( + final DataHubPolicyInfo policy, + final Optional resource) { final List users = new ArrayList<>(); final List groups = new ArrayList<>(); boolean allUsers = false; @@ -118,21 +109,50 @@ public PolicyActors getMatchingActors(final DataHubPolicyInfo policy, final Opti // 2. Fetch Actors based on resource ownership. if (actorFilter.isResourceOwners() && resource.isPresent()) { - try { - final Ownership ownership = _ownershipClient.getLatestOwnership(resource.get().getResource()); - if (ownership != null) { - users.addAll(userOwners(ownership)); - groups.addAll(groupOwners(ownership)); - } - } catch (RemoteInvocationException e) { - // Throw an error, as we are not able to fully resolve the authorized policy actors. - throw new RuntimeException("Failed to retrieve ownership when resolving authorized actors.", e); - } + Set owners = resource.get().getOwners(); + users.addAll(userOwners(owners)); + groups.addAll(groupOwners(owners)); } } return new PolicyActors(users, groups, allUsers, allGroups); } + private boolean isPolicyApplicable( + final DataHubPolicyInfo policy, + final Urn actor, + final Optional resource, + final PolicyEvaluationContext context + ) { + // If policy is inactive, simply return DENY. + if (PoliciesConfig.INACTIVE_POLICY_STATE.equals(policy.getState())) { + return false; + } + + // If the resource is not in scope, deny the request. + if (!isResourceMatch(policy.getType(), policy.getResources(), resource, context)) { + return false; + } + + // If the actor does not match, deny the request. + if (!isActorMatch(actor, policy.getActors(), resource, context)) { + return false; + } + + return true; + } + + public List getGrantedPrivileges( + final List policies, + final Urn actor, + final Optional resource) { + PolicyEvaluationContext context = new PolicyEvaluationContext(); + return policies.stream() + .filter(policy -> isPolicyApplicable(policy, actor, resource, context)) + .flatMap(policy -> policy.getPrivileges().stream()) + .distinct() + .collect(Collectors.toList()); + } + /** * Returns true if the policy matches the resource spec, false otherwise. * @@ -140,8 +160,9 @@ public PolicyActors getMatchingActors(final DataHubPolicyInfo policy, final Opti * If the policy is of type "METADATA", the resourceSpec parameter will be matched against the * resource filter defined on the policy. */ - public Boolean policyMatchesResource(final DataHubPolicyInfo policy, final Optional resourceSpec) { - return isResourceMatch(policy.getType(), policy.getResources(), resourceSpec, new PolicyEvaluationContext()); + public Boolean policyMatchesResource(final DataHubPolicyInfo policy, final Optional resourceSpec) { + return isResourceMatch(policy.getType(), policy.getResources(), resourceSpec, + new PolicyEvaluationContext()); } /** @@ -160,7 +181,7 @@ private boolean isPrivilegeMatch( private boolean isResourceMatch( final String policyType, final @Nullable DataHubResourceFilter policyResourceFilter, - final Optional requestResource, + final Optional requestResource, final PolicyEvaluationContext context) { if (PoliciesConfig.PLATFORM_POLICY_TYPE.equals(policyType)) { // Currently, platform policies have no associated resource. @@ -174,15 +195,56 @@ private boolean isResourceMatch( // Resource filter present in policy, but no resource spec provided. return false; } - final ResourceSpec resourceSpec = requestResource.get(); - final boolean resourceTypesMatch = policyResourceFilter.getType() != null - && policyResourceFilter.getType().equals(resourceSpec.getType()); - final boolean resourceIdentityMatch = - policyResourceFilter.isAllResources() - || (policyResourceFilter.getResources() != null && policyResourceFilter.getResources().stream() - .anyMatch(resource -> resource.equals(resourceSpec.getResource()))); - // If the resource's type and identity match, then the resource matches the policy. - return resourceTypesMatch && resourceIdentityMatch; + final PolicyMatchFilter filter = getFilter(policyResourceFilter); + return checkFilter(filter, requestResource.get()); + } + + /** + * Get filter object from policy resource filter. Make sure it is backward compatible by constructing PolicyMatchFilter object + * from other fields if the filter field is not set + */ + private PolicyMatchFilter getFilter(DataHubResourceFilter policyResourceFilter) { + if (policyResourceFilter.hasFilter()) { + return policyResourceFilter.getFilter(); + } + PolicyMatchCriterionArray criteria = new PolicyMatchCriterionArray(); + if (policyResourceFilter.hasType()) { + criteria.add(new PolicyMatchCriterion().setField(ResourceFieldType.RESOURCE_TYPE.name()) + .setValues(new StringArray(Collections.singletonList(policyResourceFilter.getType())))); + } + if (policyResourceFilter.hasType() && policyResourceFilter.hasResources() + && !policyResourceFilter.isAllResources()) { + criteria.add( + new PolicyMatchCriterion().setField(ResourceFieldType.RESOURCE_URN.name()).setValues(policyResourceFilter.getResources())); + } + return new PolicyMatchFilter().setCriteria(criteria); + } + + private boolean checkFilter(final PolicyMatchFilter filter, final ResolvedResourceSpec resource) { + return filter.getCriteria().stream().allMatch(criterion -> checkCriterion(criterion, resource)); + } + + private boolean checkCriterion(final PolicyMatchCriterion criterion, final ResolvedResourceSpec resource) { + ResourceFieldType resourceFieldType; + try { + resourceFieldType = ResourceFieldType.valueOf(criterion.getField().toUpperCase()); + } catch (IllegalArgumentException e) { + log.error("Unsupported field type {}", criterion.getField()); + return false; + } + + Set fieldValues = resource.getFieldValues(resourceFieldType); + return criterion.getValues() + .stream() + .anyMatch(filterValue -> checkCondition(fieldValues, filterValue, criterion.getCondition())); + } + + private boolean checkCondition(Set fieldValues, String filterValue, PolicyMatchCondition condition) { + if (condition == PolicyMatchCondition.EQUALS) { + return fieldValues.contains(filterValue); + } + log.error("Unsupported condition {}", condition); + return false; } /** @@ -191,13 +253,13 @@ private boolean isResourceMatch( private boolean isActorMatch( final Urn actor, final DataHubActorFilter actorFilter, - final Optional resourceSpec, + final Optional resourceSpec, final PolicyEvaluationContext context) { // 1. If the actor is a matching "User" in the actor filter, return true immediately. - if (isUserMatch(actor, actorFilter)) { - return true; - } + if (isUserMatch(actor, actorFilter)) { + return true; + } // 2. If the actor is in a matching "Group" in the actor filter, return true immediately. if (isGroupMatch(actor, actorFilter, context)) { @@ -230,44 +292,33 @@ private boolean isGroupMatch(final Urn actor, final DataHubActorFilter actorFilt private boolean isOwnerMatch( final Urn actor, final DataHubActorFilter actorFilter, - final Optional requestResource, + final Optional requestResource, final PolicyEvaluationContext context) { - // If the policy does not apply to owners, or there is no resource to own, return false immediately. if (!actorFilter.isResourceOwners() || !requestResource.isPresent()) { return false; } - - // Otherwise, evaluate ownership match. - final ResourceSpec resourceSpec = requestResource.get(); - try { - final Ownership ownership = _ownershipClient.getLatestOwnership(resourceSpec.getResource()); - if (ownership != null) { - return isActorOwner(actor, ownership, context); - } - } catch (Exception e) { - log.error(String.format("Failed to resolve Ownership of resource with URN %s. Returning DENY.", resourceSpec.getResource()), e); - } - return false; + return isActorOwner(actor, requestResource.get(), context); } - private boolean isActorOwner(Urn actor, Ownership ownership, PolicyEvaluationContext context) { - if (isUserOwner(actor, ownership)) { + private boolean isActorOwner(Urn actor, ResolvedResourceSpec resourceSpec, PolicyEvaluationContext context) { + Set owners = resourceSpec.getOwners(); + if (isUserOwner(actor, owners)) { return true; } final Set groups = resolveGroups(actor, context); - if (isGroupOwner(groups, ownership)) { + if (isGroupOwner(groups, owners)) { return true; } return false; } - private boolean isUserOwner(Urn actor, Ownership ownership) { - return ownership.getOwners().stream().anyMatch(owner -> actor.equals(owner.getOwner())); + private boolean isUserOwner(Urn actor, Set owners) { + return owners.contains(actor.toString()); } - private boolean isGroupOwner(Set groups, Ownership ownership) { - return ownership.getOwners().stream().anyMatch(owner -> groups.contains(owner.getOwner())); + private boolean isGroupOwner(Set groups, Set owners) { + return groups.stream().anyMatch(group -> owners.contains(group.toString())); } private Set resolveGroups(Urn actor, PolicyEvaluationContext context) { @@ -360,19 +411,17 @@ public Boolean allGroups() { } } - private List userOwners(final Ownership ownership) { - return ownership.getOwners() - .stream() - .filter(owner -> CORP_USER_ENTITY_NAME.equals(owner.getOwner().getEntityType())) - .map(Owner::getOwner) + private List userOwners(final Set owners) { + return owners.stream() + .map(UrnUtils::getUrn) + .filter(owner -> CORP_USER_ENTITY_NAME.equals(owner.getEntityType())) .collect(Collectors.toList()); } - private List groupOwners(final Ownership ownership) { - return ownership.getOwners() - .stream() - .filter(owner -> CORP_GROUP_ENTITY_NAME.equals(owner.getOwner().getEntityType())) - .map(Owner::getOwner) + private List groupOwners(final Set owners) { + return owners.stream() + .map(UrnUtils::getUrn) + .filter(owner -> CORP_GROUP_ENTITY_NAME.equals(owner.getEntityType())) .collect(Collectors.toList()); } } diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java new file mode 100644 index 00000000000000..2c7bdfe37d36e5 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResolvedResourceSpec.java @@ -0,0 +1,33 @@ +package com.datahub.authorization; + +import com.datahub.authorization.fieldresolverprovider.ResourceFieldType; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + + +/** + * Wrapper around authorization request with field resolvers for lazily fetching the field values for each field type + */ +@RequiredArgsConstructor +public class ResolvedResourceSpec { + @Getter + private final ResourceSpec spec; + private final Map fieldResolvers; + + public Set getFieldValues(ResourceFieldType resourceFieldType) { + if (!fieldResolvers.containsKey(resourceFieldType)) { + return Collections.emptySet(); + } + return fieldResolvers.get(resourceFieldType).getFieldValuesFuture().join().getValues(); + } + + public Set getOwners() { + if (!fieldResolvers.containsKey(ResourceFieldType.OWNER)) { + return Collections.emptySet(); + } + return fieldResolvers.get(ResourceFieldType.OWNER).getFieldValuesFuture().join().getValues(); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResourceSpecResolver.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResourceSpecResolver.java new file mode 100644 index 00000000000000..e0a0ff33757869 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/ResourceSpecResolver.java @@ -0,0 +1,36 @@ +package com.datahub.authorization; + +import com.datahub.authentication.Authentication; +import com.datahub.authorization.fieldresolverprovider.DomainFieldResolverProvider; +import com.datahub.authorization.fieldresolverprovider.EntityTypeFieldResolverProvider; +import com.datahub.authorization.fieldresolverprovider.EntityUrnFieldResolverProvider; +import com.datahub.authorization.fieldresolverprovider.ResourceFieldType; +import com.datahub.authorization.fieldresolverprovider.OwnerFieldResolverProvider; +import com.datahub.authorization.fieldresolverprovider.ResourceFieldResolverProvider; +import com.google.common.collect.ImmutableList; +import com.linkedin.entity.client.EntityClient; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + + +public class ResourceSpecResolver { + private final List _resourceFieldResolverProviders; + + public ResourceSpecResolver(Authentication systemAuthentication, EntityClient entityClient) { + _resourceFieldResolverProviders = + ImmutableList.of(new EntityTypeFieldResolverProvider(), new EntityUrnFieldResolverProvider(), + new DomainFieldResolverProvider(entityClient, systemAuthentication), + new OwnerFieldResolverProvider(entityClient, systemAuthentication)); + } + + public ResolvedResourceSpec resolve(ResourceSpec resourceSpec) { + return new ResolvedResourceSpec(resourceSpec, getFieldResolvers(resourceSpec)); + } + + public Map getFieldResolvers(ResourceSpec resourceSpec) { + return _resourceFieldResolverProviders.stream() + .collect(Collectors.toMap(ResourceFieldResolverProvider::getFieldType, + hydrator -> hydrator.getFieldResolver(resourceSpec))); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java new file mode 100644 index 00000000000000..fbc7fcae88d038 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/DomainFieldResolverProvider.java @@ -0,0 +1,58 @@ +package com.datahub.authorization.fieldresolverprovider; + +import com.datahub.authentication.Authentication; +import com.datahub.authorization.FieldResolver; +import com.datahub.authorization.ResourceSpec; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.domain.Domains; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import java.util.Collections; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + + +/** + * Provides field resolver for domain given resourceSpec + */ +@Slf4j +@RequiredArgsConstructor +public class DomainFieldResolverProvider implements ResourceFieldResolverProvider { + + private final EntityClient _entityClient; + private final Authentication _systemAuthentication; + + @Override + public ResourceFieldType getFieldType() { + return ResourceFieldType.DOMAIN; + } + + @Override + public FieldResolver getFieldResolver(ResourceSpec resourceSpec) { + return FieldResolver.getResolverFromFunction(resourceSpec, this::getDomains); + } + + private FieldResolver.FieldValue getDomains(ResourceSpec resourceSpec) { + Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); + EnvelopedAspect domainsAspect; + try { + EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn, + Collections.singleton(Constants.DOMAINS_ASPECT_NAME), _systemAuthentication); + if (response == null || !response.getAspects().containsKey(Constants.DOMAINS_ASPECT_NAME)) { + return FieldResolver.emptyFieldValue(); + } + domainsAspect = response.getAspects().get(Constants.DOMAINS_ASPECT_NAME); + } catch (Exception e) { + log.error("Error while retrieving domains aspect for urn {}", entityUrn, e); + return FieldResolver.emptyFieldValue(); + } + Domains domains = new Domains(domainsAspect.getValue().data()); + return FieldResolver.FieldValue.builder() + .values(domains.getDomains().stream().map(Object::toString).collect(Collectors.toSet())) + .build(); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java new file mode 100644 index 00000000000000..9fc731b8c54655 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityTypeFieldResolverProvider.java @@ -0,0 +1,21 @@ +package com.datahub.authorization.fieldresolverprovider; + +import com.datahub.authorization.FieldResolver; +import com.datahub.authorization.ResourceSpec; +import java.util.Collections; + + +/** + * Provides field resolver for entity type given resourceSpec + */ +public class EntityTypeFieldResolverProvider implements ResourceFieldResolverProvider { + @Override + public ResourceFieldType getFieldType() { + return ResourceFieldType.RESOURCE_TYPE; + } + + @Override + public FieldResolver getFieldResolver(ResourceSpec resourceSpec) { + return FieldResolver.getResolverFromValues(Collections.singleton(resourceSpec.getType())); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java new file mode 100644 index 00000000000000..4b548e61a440b5 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/EntityUrnFieldResolverProvider.java @@ -0,0 +1,21 @@ +package com.datahub.authorization.fieldresolverprovider; + +import com.datahub.authorization.FieldResolver; +import com.datahub.authorization.ResourceSpec; +import java.util.Collections; + + +/** + * Provides field resolver for entity urn given resourceSpec + */ +public class EntityUrnFieldResolverProvider implements ResourceFieldResolverProvider { + @Override + public ResourceFieldType getFieldType() { + return ResourceFieldType.RESOURCE_URN; + } + + @Override + public FieldResolver getFieldResolver(ResourceSpec resourceSpec) { + return FieldResolver.getResolverFromValues(Collections.singleton(resourceSpec.getResource())); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java new file mode 100644 index 00000000000000..73438f53638533 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/OwnerFieldResolverProvider.java @@ -0,0 +1,58 @@ +package com.datahub.authorization.fieldresolverprovider; + +import com.datahub.authentication.Authentication; +import com.datahub.authorization.FieldResolver; +import com.datahub.authorization.ResourceSpec; +import com.linkedin.common.Ownership; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import java.util.Collections; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + + +/** + * Provides field resolver for owners given resourceSpec + */ +@Slf4j +@RequiredArgsConstructor +public class OwnerFieldResolverProvider implements ResourceFieldResolverProvider { + + private final EntityClient _entityClient; + private final Authentication _systemAuthentication; + + @Override + public ResourceFieldType getFieldType() { + return ResourceFieldType.OWNER; + } + + @Override + public FieldResolver getFieldResolver(ResourceSpec resourceSpec) { + return FieldResolver.getResolverFromFunction(resourceSpec, this::getOwners); + } + + private FieldResolver.FieldValue getOwners(ResourceSpec resourceSpec) { + Urn entityUrn = UrnUtils.getUrn(resourceSpec.getResource()); + EnvelopedAspect ownershipAspect; + try { + EntityResponse response = _entityClient.getV2(entityUrn.getEntityType(), entityUrn, + Collections.singleton(Constants.OWNERSHIP_ASPECT_NAME), _systemAuthentication); + if (response == null || !response.getAspects().containsKey(Constants.OWNERSHIP_ASPECT_NAME)) { + return FieldResolver.emptyFieldValue(); + } + ownershipAspect = response.getAspects().get(Constants.OWNERSHIP_ASPECT_NAME); + } catch (Exception e) { + log.error("Error while retrieving domains aspect for urn {}", entityUrn, e); + return FieldResolver.emptyFieldValue(); + } + Ownership ownership = new Ownership(ownershipAspect.getValue().data()); + return FieldResolver.FieldValue.builder() + .values(ownership.getOwners().stream().map(owner -> owner.getOwner().toString()).collect(Collectors.toSet())) + .build(); + } +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java new file mode 100644 index 00000000000000..2c3b3157e359a0 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldResolverProvider.java @@ -0,0 +1,21 @@ +package com.datahub.authorization.fieldresolverprovider; + +import com.datahub.authorization.FieldResolver; +import com.datahub.authorization.ResourceSpec; + + +/** + * Base class for defining a class that provides the field resolver for the given field type + */ +public interface ResourceFieldResolverProvider { + + /** + * Field that this hydrator is hydrating + */ + ResourceFieldType getFieldType(); + + /** + * Return resolver for fetching the field values given the resource + */ + FieldResolver getFieldResolver(ResourceSpec resourceSpec); +} diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldType.java b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldType.java new file mode 100644 index 00000000000000..856f4df7db8f48 --- /dev/null +++ b/metadata-service/auth-impl/src/main/java/com/datahub/authorization/fieldresolverprovider/ResourceFieldType.java @@ -0,0 +1,23 @@ +package com.datahub.authorization.fieldresolverprovider; + +/** + * List of resource field types to fetch for a given resource + */ +public enum ResourceFieldType { + /** + * Type of resource (e.g. dataset, chart) + */ + RESOURCE_TYPE, + /** + * Urn of resource + */ + RESOURCE_URN, + /** + * Owners of resource + */ + OWNER, + /** + * Domains of resource + */ + DOMAIN +} diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/AuthorizationManagerTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/AuthorizationManagerTest.java index e63502c0f0ba03..891b69415a4468 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/AuthorizationManagerTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/AuthorizationManagerTest.java @@ -5,6 +5,7 @@ import com.datahub.authentication.Authentication; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.Owner; import com.linkedin.common.OwnerArray; @@ -18,7 +19,6 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.OwnershipClient; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.policy.DataHubActorFilter; import com.linkedin.policy.DataHubPolicyInfo; @@ -26,16 +26,23 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; -import org.mockito.Mockito; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.authorization.PoliciesConfig.*; -import static org.mockito.Mockito.*; -import static org.testng.Assert.*; +import static com.linkedin.metadata.Constants.DATAHUB_POLICY_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; +import static com.linkedin.metadata.Constants.POLICY_ENTITY_NAME; +import static com.linkedin.metadata.authorization.PoliciesConfig.ACTIVE_POLICY_STATE; +import static com.linkedin.metadata.authorization.PoliciesConfig.INACTIVE_POLICY_STATE; +import static com.linkedin.metadata.authorization.PoliciesConfig.METADATA_POLICY_TYPE; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; public class AuthorizationManagerTest { @@ -47,7 +54,7 @@ public class AuthorizationManagerTest { @BeforeMethod public void setupTest() throws Exception { - _entityClient = Mockito.mock(EntityClient.class); + _entityClient = mock(EntityClient.class); // Init mocks. final Urn activePolicyUrn = Urn.createFromString("urn:li:dataHubPolicy:0"); @@ -85,10 +92,8 @@ inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(ina envelopedAspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect() .setValue(new com.linkedin.entity.Aspect(createOwnershipAspect(userUrns, groupUrns).data()))); entityResponse.setAspects(envelopedAspectMap); - Map mockMap = mock(Map.class); - when(_entityClient.batchGetV2(any(), any(), eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), any())) - .thenReturn(mockMap); - when(mockMap.get(any(Urn.class))).thenReturn(entityResponse); + when(_entityClient.getV2(any(), any(), eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), any())) + .thenReturn(entityResponse); final Authentication systemAuthentication = new Authentication( new Actor(ActorType.USER, DATAHUB_SYSTEM_CLIENT_ID), @@ -98,7 +103,6 @@ inactivePolicyUrn, new EntityResponse().setUrn(inactivePolicyUrn).setAspects(ina _authorizationManager = new AuthorizationManager( systemAuthentication, _entityClient, - new OwnershipClient(_entityClient), 10, 10, Authorizer.AuthorizationMode.DEFAULT @@ -205,23 +209,21 @@ public void testInvalidateCache() throws Exception { @Test public void testAuthorizedActorsActivePolicy() throws Exception { - - final AuthorizationManager.AuthorizedActors actors = _authorizationManager.authorizedActors( - "EDIT_ENTITY_TAGS", // Should be inside the active policy. - Optional.of(new ResourceSpec("dataset", "urn:li:dataset:1")) - ); + final AuthorizationManager.AuthorizedActors actors = + _authorizationManager.authorizedActors("EDIT_ENTITY_TAGS", // Should be inside the active policy. + Optional.of(new ResourceSpec("dataset", "urn:li:dataset:1"))); assertTrue(actors.allUsers()); assertTrue(actors.allGroups()); - assertEquals(actors.getUsers(), ImmutableList.of( + assertEquals(new HashSet<>(actors.getUsers()), ImmutableSet.of( Urn.createFromString("urn:li:corpuser:user1"), Urn.createFromString("urn:li:corpuser:user2"), Urn.createFromString("urn:li:corpuser:user3"), Urn.createFromString("urn:li:corpuser:user4") )); - assertEquals(actors.getGroups(), ImmutableList.of( + assertEquals(new HashSet<>(actors.getGroups()), ImmutableSet.of( Urn.createFromString("urn:li:corpGroup:group1"), Urn.createFromString("urn:li:corpGroup:group2"), Urn.createFromString("urn:li:corpGroup:group3"), diff --git a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java index 9d31f66594d2e1..9064a9f7839a71 100644 --- a/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java +++ b/metadata-service/auth-impl/src/test/java/com/datahub/authorization/PolicyEngineTest.java @@ -1,7 +1,10 @@ package com.datahub.authorization; import com.datahub.authentication.Authentication; +import com.datahub.authorization.fieldresolverprovider.ResourceFieldType; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; import com.linkedin.common.Owner; import com.linkedin.common.OwnerArray; @@ -9,13 +12,13 @@ import com.linkedin.common.OwnershipType; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.StringArray; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; -import com.linkedin.entity.client.OwnershipClient; import com.linkedin.identity.CorpUserInfo; import com.linkedin.identity.GroupMembership; import com.linkedin.policy.DataHubActorFilter; @@ -23,16 +26,31 @@ import com.linkedin.policy.DataHubResourceFilter; import java.net.URISyntaxException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import org.mockito.Mockito; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import static com.linkedin.metadata.Constants.*; -import static com.linkedin.metadata.authorization.PoliciesConfig.*; -import static org.mockito.Mockito.*; -import static org.testng.Assert.*; +import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; +import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME; +import static com.linkedin.metadata.Constants.OWNERSHIP_ASPECT_NAME; +import static com.linkedin.metadata.authorization.PoliciesConfig.ACTIVE_POLICY_STATE; +import static com.linkedin.metadata.authorization.PoliciesConfig.INACTIVE_POLICY_STATE; +import static com.linkedin.metadata.authorization.PoliciesConfig.METADATA_POLICY_TYPE; +import static com.linkedin.metadata.authorization.PoliciesConfig.PLATFORM_POLICY_TYPE; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; public class PolicyEngineTest { @@ -44,6 +62,8 @@ public class PolicyEngineTest { private static final String RESOURCE_URN = "urn:li:dataset:test"; + private static final String DOMAIN_URN = "urn:li:domain:domain1"; + private EntityClient _entityClient; private PolicyEngine _policyEngine; @@ -54,27 +74,29 @@ public class PolicyEngineTest { @BeforeMethod public void setupTest() throws Exception { _entityClient = Mockito.mock(EntityClient.class); - _policyEngine = new PolicyEngine(Mockito.mock(Authentication.class), _entityClient, new OwnershipClient(_entityClient)); + _policyEngine = new PolicyEngine(Mockito.mock(Authentication.class), _entityClient); // Init mocks. EntityResponse authorizedEntityResponse = createAuthorizedEntityResponse(); authorizedUserUrn = Urn.createFromString(AUTHORIZED_PRINCIPAL); authorizedEntityResponse.setUrn(authorizedUserUrn); - Map authorizedEntityResponseMap = Collections.singletonMap(authorizedUserUrn, authorizedEntityResponse); - when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), - eq(null), any())).thenReturn(authorizedEntityResponseMap); + Map authorizedEntityResponseMap = + Collections.singletonMap(authorizedUserUrn, authorizedEntityResponse); + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), eq(null), + any())).thenReturn(authorizedEntityResponseMap); EntityResponse unauthorizedEntityResponse = createUnauthorizedEntityResponse(); unauthorizedUserUrn = Urn.createFromString(UNAUTHORIZED_PRINCIPAL); unauthorizedEntityResponse.setUrn(unauthorizedUserUrn); - Map unauthorizedEntityResponseMap = Collections.singletonMap(unauthorizedUserUrn, unauthorizedEntityResponse); - when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), - eq(null), any())).thenReturn(unauthorizedEntityResponseMap); + Map unauthorizedEntityResponseMap = + Collections.singletonMap(unauthorizedUserUrn, unauthorizedEntityResponse); + when(_entityClient.batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), eq(null), + any())).thenReturn(unauthorizedEntityResponseMap); EntityResponse entityResponse = new EntityResponse(); EnvelopedAspectMap envelopedAspectMap = new EnvelopedAspectMap(); - envelopedAspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect() - .setValue(new com.linkedin.entity.Aspect(createOwnershipAspect(true, true).data()))); + envelopedAspectMap.put(OWNERSHIP_ASPECT_NAME, + new EnvelopedAspect().setValue(new com.linkedin.entity.Aspect(createOwnershipAspect(true, true).data()))); entityResponse.setAspects(envelopedAspectMap); resourceUrn = Urn.createFromString(RESOURCE_URN); Map mockMap = mock(Map.class); @@ -106,15 +128,10 @@ public void testEvaluatePolicyInactivePolicyState() { resourceFilter.setAllResources(true); resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); - - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertFalse(result.isGranted()); } @@ -140,20 +157,14 @@ public void testEvaluatePolicyPrivilegeFilterNoMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_OWNERS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_OWNERS", + Optional.of(resourceSpec)); assertFalse(result.isGranted()); // Verify no network calls - verify(_entityClient, times(0)).batchGetV2( - any(), any(), any(), any()); + verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); } @Test @@ -172,17 +183,12 @@ public void testEvaluatePlatformPolicyPrivilegeFilterMatch() throws Exception { actorFilter.setAllGroups(false); dataHubPolicyInfo.setActors(actorFilter); - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - AUTHORIZED_PRINCIPAL, - "MANAGE_POLICIES", - Optional.empty() - ); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "MANAGE_POLICIES", Optional.empty()); assertTrue(result.isGranted()); // Verify no network calls - verify(_entityClient, times(0)).batchGetV2( - any(), any(), any(), any()); + verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); } @Test @@ -210,15 +216,11 @@ public void testEvaluatePolicyActorFilterUserMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert Authorized user can edit entity tags. - PolicyEngine.PolicyEvaluationResult result1 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result1.isGranted()); @@ -251,16 +253,11 @@ public void testEvaluatePolicyActorFilterUserNoMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert unauthorized user cannot edit entity tags. - PolicyEngine.PolicyEvaluationResult result2 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - "urn:li:corpuser:test", - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result2 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, "urn:li:corpuser:test", "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertFalse(result2.isGranted()); @@ -293,20 +290,16 @@ public void testEvaluatePolicyActorFilterGroupMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert authorized user can edit entity tags, because of group membership. - PolicyEngine.PolicyEvaluationResult result1 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result1.isGranted()); // Verify we are only calling for group during these requests. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), - eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); + verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + eq(null), any()); } @Test @@ -334,16 +327,11 @@ public void testEvaluatePolicyActorFilterGroupNoMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert unauthorized user cannot edit entity tags. - PolicyEngine.PolicyEvaluationResult result2 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - UNAUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result2 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertFalse(result2.isGranted()); // Verify we are only calling for group during these requests. @@ -373,27 +361,17 @@ public void testEvaluatePolicyActorFilterAllUsersMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert authorized user can edit entity tags, because of group membership. - PolicyEngine.PolicyEvaluationResult result1 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result1.isGranted()); // Assert unauthorized user cannot edit entity tags. - PolicyEngine.PolicyEvaluationResult result2 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - UNAUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result2 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result2.isGranted()); // Verify no network calls @@ -422,32 +400,22 @@ public void testEvaluatePolicyActorFilterAllGroupsMatch() throws Exception { resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert authorized user can edit entity tags, because of group membership. - PolicyEngine.PolicyEvaluationResult result1 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result1.isGranted()); // Assert unauthorized user cannot edit entity tags. - PolicyEngine.PolicyEvaluationResult result2 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - UNAUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result2 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result2.isGranted()); // Verify we are only calling for group during these requests. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), - eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); + verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + eq(null), any()); verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), eq(null), any()); } @@ -474,29 +442,17 @@ public void testEvaluatePolicyActorFilterUserResourceOwnersMatch() throws Except resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL), Collections.emptySet()); // Assert authorized user can edit entity tags, because he is a user owner. - PolicyEngine.PolicyEvaluationResult result1 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result1.isGranted()); - // Verify we are calling for the resource ownership aspect - verify(_entityClient, times(1)).batchGetV2( - any(), - eq(Collections.singleton(resourceUrn)), - eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), - any()); // Ensure no calls for group membership. - verify(_entityClient, times(0)).batchGetV2( - eq(CORP_USER_ENTITY_NAME), - eq(Collections.singleton(authorizedUserUrn)), - eq(null), - any()); + verify(_entityClient, times(0)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + eq(null), any()); } @Test @@ -521,37 +477,17 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersMatch() throws Excep resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); - // Overwrite the Ownership of the Resource to only include a single group. - EntityResponse entityResponse = new EntityResponse(); - EnvelopedAspectMap envelopedAspectMap = new EnvelopedAspectMap(); - envelopedAspectMap.put(OWNERSHIP_ASPECT_NAME, new EnvelopedAspect() - .setValue(new com.linkedin.entity.Aspect(createOwnershipAspect(false, true).data()))); - entityResponse.setAspects(envelopedAspectMap); - Map mockMap = mock(Map.class); - when(_entityClient.batchGetV2(any(), eq(Collections.singleton(resourceUrn)), - eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), any())).thenReturn(mockMap); - when(mockMap.get(any(Urn.class))).thenReturn(entityResponse); - + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_GROUP), Collections.emptySet()); // Assert authorized user can edit entity tags, because he is a user owner. - PolicyEngine.PolicyEvaluationResult result1 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result1 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result1.isGranted()); - // Verify we are calling for the resource ownership aspect - verify(_entityClient, times(1)).batchGetV2( - any(), - eq(Collections.singleton(resourceUrn)), - eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), - any()); // Ensure that caching of groups is working with 1 call to entity client for each principal. - verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), - eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); + verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + eq(null), any()); } @Test @@ -575,24 +511,13 @@ public void testEvaluatePolicyActorFilterGroupResourceOwnersNoMatch() throws Exc resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); // Assert unauthorized user cannot edit entity tags. - PolicyEngine.PolicyEvaluationResult result2 = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - UNAUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + PolicyEngine.PolicyEvaluationResult result2 = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, UNAUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertFalse(result2.isGranted()); - // Verify we are calling for the resource ownership aspect - verify(_entityClient, times(1)).batchGetV2( - any(), - eq(Collections.singleton(resourceUrn)), - eq(Collections.singleton(OWNERSHIP_ASPECT_NAME)), - any()); // Ensure that caching of groups is working with 1 call to entity client for each principal. verify(_entityClient, times(1)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(unauthorizedUserUrn)), eq(null), any()); @@ -619,15 +544,11 @@ public void testEvaluatePolicyResourceFilterAllResourcesMatch() throws Exception resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - "urn:li:dataset:random" // A dataset Authorized principal _does not own_. - )) - ); + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", "urn:li:dataset:random"); // A dataset Authorized principal _does not own_. + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result.isGranted()); // Verify no network calls @@ -655,15 +576,10 @@ public void testEvaluatePolicyResourceFilterAllResourcesNoMatch() throws Excepti resourceFilter.setType("dataset"); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "chart", // Notice: Not a dataset. - RESOURCE_URN - )) - ); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("chart", RESOURCE_URN); // Notice: Not a dataset. + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertFalse(result.isGranted()); // Verify no network calls @@ -671,7 +587,7 @@ public void testEvaluatePolicyResourceFilterAllResourcesNoMatch() throws Excepti } @Test - public void testEvaluatePolicyResourceFilterSpecificResourceMatch() throws Exception { + public void testEvaluatePolicyResourceFilterSpecificResourceMatchLegacy() throws Exception { final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); @@ -695,15 +611,42 @@ public void testEvaluatePolicyResourceFilterSpecificResourceMatch() throws Excep resourceFilter.setResources(resourceUrns); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN - )) - ); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); + assertTrue(result.isGranted()); + + // Verify no network calls + verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); + } + + @Test + public void testEvaluatePolicyResourceFilterSpecificResourceMatch() throws Exception { + final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); + dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo.setPrivileges(new StringArray("EDIT_ENTITY_TAGS")); + dataHubPolicyInfo.setDisplayName("My Test Display"); + dataHubPolicyInfo.setDescription("My test display!"); + dataHubPolicyInfo.setEditable(true); + + final DataHubActorFilter actorFilter = new DataHubActorFilter(); + actorFilter.setResourceOwners(true); + actorFilter.setAllUsers(true); + actorFilter.setAllGroups(true); + dataHubPolicyInfo.setActors(actorFilter); + + final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); + resourceFilter.setFilter(FilterUtils.newFilter( + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.RESOURCE_URN, + Collections.singletonList(RESOURCE_URN)))); + dataHubPolicyInfo.setResources(resourceFilter); + + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertTrue(result.isGranted()); // Verify no network calls @@ -727,29 +670,189 @@ public void testEvaluatePolicyResourceFilterSpecificResourceNoMatch() throws Exc dataHubPolicyInfo.setActors(actorFilter); final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); - resourceFilter.setAllResources(false); - resourceFilter.setType("dataset"); + resourceFilter.setFilter(FilterUtils.newFilter( + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.RESOURCE_URN, + Collections.singletonList(RESOURCE_URN)))); + dataHubPolicyInfo.setResources(resourceFilter); - StringArray resourceUrns = new StringArray(); - resourceUrns.add(RESOURCE_URN); // Filter applies to specific resource. - resourceFilter.setResources(resourceUrns); + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy. + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); + assertFalse(result.isGranted()); + + // Verify no network calls + verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); + } + + @Test + public void testEvaluatePolicyResourceFilterSpecificResourceMatchDomain() throws Exception { + final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); + dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo.setPrivileges(new StringArray("EDIT_ENTITY_TAGS")); + dataHubPolicyInfo.setDisplayName("My Test Display"); + dataHubPolicyInfo.setDescription("My test display!"); + dataHubPolicyInfo.setEditable(true); + + final DataHubActorFilter actorFilter = new DataHubActorFilter(); + actorFilter.setResourceOwners(true); + actorFilter.setAllUsers(true); + actorFilter.setAllGroups(true); + dataHubPolicyInfo.setActors(actorFilter); + + final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); + resourceFilter.setFilter(FilterUtils.newFilter( + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.DOMAIN, + Collections.singletonList(DOMAIN_URN)))); + dataHubPolicyInfo.setResources(resourceFilter); + + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(), Collections.singleton(DOMAIN_URN)); + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); + assertTrue(result.isGranted()); + + // Verify no network calls + verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); + } + + @Test + public void testEvaluatePolicyResourceFilterSpecificResourceNoMatchDomain() throws Exception { + final DataHubPolicyInfo dataHubPolicyInfo = new DataHubPolicyInfo(); + dataHubPolicyInfo.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo.setPrivileges(new StringArray("EDIT_ENTITY_TAGS")); + dataHubPolicyInfo.setDisplayName("My Test Display"); + dataHubPolicyInfo.setDescription("My test display!"); + dataHubPolicyInfo.setEditable(true); + + final DataHubActorFilter actorFilter = new DataHubActorFilter(); + actorFilter.setResourceOwners(true); + actorFilter.setAllUsers(true); + actorFilter.setAllGroups(true); + dataHubPolicyInfo.setActors(actorFilter); + + final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); + resourceFilter.setFilter(FilterUtils.newFilter( + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.DOMAIN, + Collections.singletonList(DOMAIN_URN)))); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyEvaluationResult result = _policyEngine.evaluatePolicy( - dataHubPolicyInfo, - AUTHORIZED_PRINCIPAL, - "EDIT_ENTITY_TAGS", - Optional.of(new ResourceSpec( - "dataset", - "urn:li:dataset:random" // A resource not covered by the policy. - )) - ); + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(), + Collections.singleton("urn:li:domain:domain2")); // Domain doesn't match + PolicyEngine.PolicyEvaluationResult result = + _policyEngine.evaluatePolicy(dataHubPolicyInfo, AUTHORIZED_PRINCIPAL, "EDIT_ENTITY_TAGS", + Optional.of(resourceSpec)); assertFalse(result.isGranted()); // Verify no network calls verify(_entityClient, times(0)).batchGetV2(any(), any(), any(), any()); } + @Test + public void testGetGrantedPrivileges() throws Exception { + // Policy 1, match dataset type and domain + final DataHubPolicyInfo dataHubPolicyInfo1 = new DataHubPolicyInfo(); + dataHubPolicyInfo1.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo1.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo1.setPrivileges(new StringArray("PRIVILEGE_1")); + dataHubPolicyInfo1.setDisplayName("My Test Display"); + dataHubPolicyInfo1.setDescription("My test display!"); + dataHubPolicyInfo1.setEditable(true); + + final DataHubActorFilter actorFilter1 = new DataHubActorFilter(); + actorFilter1.setResourceOwners(true); + actorFilter1.setAllUsers(true); + actorFilter1.setAllGroups(true); + dataHubPolicyInfo1.setActors(actorFilter1); + + final DataHubResourceFilter resourceFilter1 = new DataHubResourceFilter(); + resourceFilter1.setFilter(FilterUtils.newFilter( + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.DOMAIN, + Collections.singletonList(DOMAIN_URN)))); + dataHubPolicyInfo1.setResources(resourceFilter1); + + // Policy 2, match dataset type and resource + final DataHubPolicyInfo dataHubPolicyInfo2 = new DataHubPolicyInfo(); + dataHubPolicyInfo2.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo2.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo2.setPrivileges(new StringArray("PRIVILEGE_2_1", "PRIVILEGE_2_2")); + dataHubPolicyInfo2.setDisplayName("My Test Display"); + dataHubPolicyInfo2.setDescription("My test display!"); + dataHubPolicyInfo2.setEditable(true); + + final DataHubActorFilter actorFilter2 = new DataHubActorFilter(); + actorFilter2.setResourceOwners(true); + actorFilter2.setAllUsers(true); + actorFilter2.setAllGroups(true); + dataHubPolicyInfo2.setActors(actorFilter2); + + final DataHubResourceFilter resourceFilter2 = new DataHubResourceFilter(); + resourceFilter2.setFilter(FilterUtils.newFilter( + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, Collections.singletonList("dataset"), ResourceFieldType.RESOURCE_URN, + Collections.singletonList(RESOURCE_URN)))); + dataHubPolicyInfo2.setResources(resourceFilter2); + + // Policy 3, match dataset type and owner (legacy resource filter) + final DataHubPolicyInfo dataHubPolicyInfo3 = new DataHubPolicyInfo(); + dataHubPolicyInfo3.setType(METADATA_POLICY_TYPE); + dataHubPolicyInfo3.setState(ACTIVE_POLICY_STATE); + dataHubPolicyInfo3.setPrivileges(new StringArray("PRIVILEGE_3")); + dataHubPolicyInfo3.setDisplayName("My Test Display"); + dataHubPolicyInfo3.setDescription("My test display!"); + dataHubPolicyInfo3.setEditable(true); + + final DataHubActorFilter actorFilter3 = new DataHubActorFilter(); + actorFilter3.setResourceOwners(true); + actorFilter3.setAllUsers(false); + actorFilter3.setAllGroups(false); + dataHubPolicyInfo3.setActors(actorFilter3); + + final DataHubResourceFilter resourceFilter3 = new DataHubResourceFilter(); + resourceFilter3.setAllResources(true); + resourceFilter3.setType("dataset"); + dataHubPolicyInfo3.setResources(resourceFilter3); + + final List policies = + ImmutableList.of(dataHubPolicyInfo1, dataHubPolicyInfo2, dataHubPolicyInfo3); + + assertEquals(_policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.empty()), + Collections.emptyList()); + + ResolvedResourceSpec resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(), + Collections.singleton(DOMAIN_URN)); // Everything matches + assertEquals( + _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)), + ImmutableList.of("PRIVILEGE_1", "PRIVILEGE_2_1", "PRIVILEGE_2_2")); + + resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.emptySet(), + Collections.singleton("urn:li:domain:domain2")); // Domain doesn't match + assertEquals( + _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)), + ImmutableList.of("PRIVILEGE_2_1", "PRIVILEGE_2_2")); + + resourceSpec = buildResourceResolvers("dataset", "urn:li:dataset:random", Collections.emptySet(), + Collections.singleton(DOMAIN_URN)); // Resource doesn't match + assertEquals( + _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)), + ImmutableList.of("PRIVILEGE_1")); + + resourceSpec = buildResourceResolvers("dataset", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL), + Collections.singleton(DOMAIN_URN)); // Is owner + assertEquals( + _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)), + ImmutableList.of("PRIVILEGE_1", "PRIVILEGE_2_1", "PRIVILEGE_2_2", "PRIVILEGE_3")); + + resourceSpec = buildResourceResolvers("chart", RESOURCE_URN, Collections.singleton(AUTHORIZED_PRINCIPAL), + Collections.singleton(DOMAIN_URN)); // Resource type doesn't match + assertEquals( + _policyEngine.getGrantedPrivileges(policies, UrnUtils.getUrn(AUTHORIZED_PRINCIPAL), Optional.of(resourceSpec)), + Collections.emptyList()); + } + @Test public void testGetMatchingActorsResourceMatch() throws Exception { @@ -765,18 +868,10 @@ public void testGetMatchingActorsResourceMatch() throws Exception { actorFilter.setResourceOwners(true); actorFilter.setAllUsers(true); actorFilter.setAllGroups(true); - actorFilter.setUsers(new UrnArray( - ImmutableList.of( - Urn.createFromString("urn:li:corpuser:user1"), - Urn.createFromString("urn:li:corpuser:user2") - ) - )); - actorFilter.setGroups(new UrnArray( - ImmutableList.of( - Urn.createFromString("urn:li:corpGroup:group1"), - Urn.createFromString("urn:li:corpGroup:group2") - ) - )); + actorFilter.setUsers(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:corpuser:user1"), + Urn.createFromString("urn:li:corpuser:user2")))); + actorFilter.setGroups(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group1"), + Urn.createFromString("urn:li:corpGroup:group2")))); dataHubPolicyInfo.setActors(actorFilter); final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); @@ -787,33 +882,26 @@ public void testGetMatchingActorsResourceMatch() throws Exception { resourceFilter.setResources(resourceUrns); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors( - dataHubPolicyInfo, - Optional.of(new ResourceSpec( - "dataset", - RESOURCE_URN // A resource covered by the policy. - )) - ); + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", RESOURCE_URN, ImmutableSet.of(AUTHORIZED_PRINCIPAL, AUTHORIZED_GROUP), + Collections.emptySet()); + PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec)); assertTrue(actors.allUsers()); assertTrue(actors.allGroups()); - assertEquals(actors.getUsers(), ImmutableList.of( - Urn.createFromString("urn:li:corpuser:user1"), - Urn.createFromString("urn:li:corpuser:user2"), - Urn.createFromString(AUTHORIZED_PRINCIPAL) // Resource Owner - )); + assertEquals(actors.getUsers(), + ImmutableList.of(Urn.createFromString("urn:li:corpuser:user1"), Urn.createFromString("urn:li:corpuser:user2"), + Urn.createFromString(AUTHORIZED_PRINCIPAL) // Resource Owner + )); - assertEquals(actors.getGroups(), ImmutableList.of( - Urn.createFromString("urn:li:corpGroup:group1"), - Urn.createFromString("urn:li:corpGroup:group2"), - Urn.createFromString(AUTHORIZED_GROUP) // Resource Owner + assertEquals(actors.getGroups(), ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group1"), + Urn.createFromString("urn:li:corpGroup:group2"), Urn.createFromString(AUTHORIZED_GROUP) // Resource Owner )); // Verify aspect client called, entity client not called. - verify(_entityClient, times(1)).batchGetV2(any(), any(), any(), any()); - verify(_entityClient, times(0)).batchGetV2(eq(CORP_USER_ENTITY_NAME), - eq(Collections.singleton(authorizedUserUrn)), eq(null), any()); + verify(_entityClient, times(0)).batchGetV2(eq(CORP_USER_ENTITY_NAME), eq(Collections.singleton(authorizedUserUrn)), + eq(null), any()); } @Test @@ -831,18 +919,10 @@ public void testGetMatchingActorsNoResourceMatch() throws Exception { actorFilter.setResourceOwners(true); actorFilter.setAllUsers(true); actorFilter.setAllGroups(true); - actorFilter.setUsers(new UrnArray( - ImmutableList.of( - Urn.createFromString("urn:li:corpuser:user1"), - Urn.createFromString("urn:li:corpuser:user2") - ) - )); - actorFilter.setGroups(new UrnArray( - ImmutableList.of( - Urn.createFromString("urn:li:corpGroup:group1"), - Urn.createFromString("urn:li:corpGroup:group2") - ) - )); + actorFilter.setUsers(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:corpuser:user1"), + Urn.createFromString("urn:li:corpuser:user2")))); + actorFilter.setGroups(new UrnArray(ImmutableList.of(Urn.createFromString("urn:li:corpGroup:group1"), + Urn.createFromString("urn:li:corpGroup:group2")))); dataHubPolicyInfo.setActors(actorFilter); final DataHubResourceFilter resourceFilter = new DataHubResourceFilter(); @@ -853,13 +933,9 @@ public void testGetMatchingActorsNoResourceMatch() throws Exception { resourceFilter.setResources(resourceUrns); dataHubPolicyInfo.setResources(resourceFilter); - PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors( - dataHubPolicyInfo, - Optional.of(new ResourceSpec( - "dataset", - "urn:li:dataset:random" // A resource not covered by the policy. - )) - ); + ResolvedResourceSpec resourceSpec = + buildResourceResolvers("dataset", "urn:li:dataset:random"); // A resource not covered by the policy. + PolicyEngine.PolicyActors actors = _policyEngine.getMatchingActors(dataHubPolicyInfo, Optional.of(resourceSpec)); assertFalse(actors.allUsers()); assertFalse(actors.allGroups()); @@ -938,4 +1014,17 @@ private EntityResponse createUnauthorizedEntityResponse() throws URISyntaxExcept entityResponse.setAspects(aspectMap); return entityResponse; } + + public static ResolvedResourceSpec buildResourceResolvers(String entityType, String entityUrn) { + return buildResourceResolvers(entityType, entityUrn, Collections.emptySet(), Collections.emptySet()); + } + + public static ResolvedResourceSpec buildResourceResolvers(String entityType, String entityUrn, Set owners, + Set domains) { + return new ResolvedResourceSpec(new ResourceSpec(entityType, entityUrn), + ImmutableMap.of(ResourceFieldType.RESOURCE_TYPE, FieldResolver.getResolverFromValues(Collections.singleton(entityType)), + ResourceFieldType.RESOURCE_URN, FieldResolver.getResolverFromValues(Collections.singleton(entityUrn)), + ResourceFieldType.OWNER, FieldResolver.getResolverFromValues(owners), ResourceFieldType.DOMAIN, + FieldResolver.getResolverFromValues(domains))); + } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizationManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizationManagerFactory.java index 0f11678abb3d79..e9ca703b8c4aba 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizationManagerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/auth/AuthorizationManagerFactory.java @@ -3,7 +3,6 @@ import com.datahub.authentication.Authentication; import com.datahub.authorization.AuthorizationManager; import com.linkedin.entity.client.JavaEntityClient; -import com.linkedin.entity.client.OwnershipClient; import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import javax.annotation.Nonnull; @@ -44,9 +43,7 @@ protected AuthorizationManager getInstance() { final AuthorizationManager.AuthorizationMode mode = policiesEnabled ? AuthorizationManager.AuthorizationMode.DEFAULT : AuthorizationManager.AuthorizationMode.ALLOW_ALL; - final OwnershipClient ownershipClient = new OwnershipClient(entityClient); - - return new AuthorizationManager(systemAuthentication, entityClient, ownershipClient, 10, + return new AuthorizationManager(systemAuthentication, entityClient, 10, policyCacheRefreshIntervalSeconds, mode); } } diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java index 55c5c3d9d6d3c3..86d46bd2137763 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java @@ -61,16 +61,23 @@ public void execute() throws IOException, URISyntaxException { // 2. For each JSON object, cast into a DataHub Policy Info object. for (Iterator it = policiesObj.iterator(); it.hasNext(); ) { final JsonNode policyObj = it.next(); - final DataHubPolicyInfo info = RecordUtils.toRecordTemplate(DataHubPolicyInfo.class, policyObj.get("info").toString()); final Urn urn = Urn.createFromString(policyObj.get("urn").asText()); + // If the info is not there, it means that the policy was there before, but must now be removed + if (!policyObj.has("info")) { + _entityService.deleteUrn(urn); + continue; + } + + final DataHubPolicyInfo info = RecordUtils.toRecordTemplate(DataHubPolicyInfo.class, policyObj.get("info").toString()); + if (!info.isEditable()) { // If the Policy is not editable, always re-ingest. log.info(String.format("Ingesting default policy with urn %s", urn)); ingestPolicy(urn, info); } else { // If the Policy is editable (ie. an example policy), only ingest on a clean boot up. - if (!hasDefaultPolicies) { + if (!hasPolicy(urn)) { log.info(String.format("Ingesting default policy with urn %s", urn)); ingestPolicy(urn, info); } else { @@ -113,4 +120,10 @@ private boolean hasDefaultPolicies() throws URISyntaxException { RecordTemplate aspect = _entityService.getAspect(defaultPolicyUrn, POLICY_INFO_ASPECT_NAME, 0); return aspect != null; } + + private boolean hasPolicy(Urn policyUrn) { + // Check if policy exists + RecordTemplate aspect = _entityService.getAspect(policyUrn, POLICY_INFO_ASPECT_NAME, 0); + return aspect != null; + } } diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index b0172cb76ae8f9..d183c9269088f3 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -4599,6 +4599,52 @@ "type" : "boolean", "doc" : "Whether the policy should be applied to all assets matching the filter.", "default" : false + }, { + "name" : "filter", + "type" : { + "type" : "record", + "name" : "PolicyMatchFilter", + "doc" : "The filter for specifying the resource or actor to apply privileges to", + "fields" : [ { + "name" : "criteria", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "PolicyMatchCriterion", + "doc" : "A criterion for matching a field with given value", + "fields" : [ { + "name" : "field", + "type" : "string", + "doc" : "The name of the field that the criterion refers to" + }, { + "name" : "values", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "Values. one of which the intended field should match condition" + }, { + "name" : "condition", + "type" : { + "type" : "enum", + "name" : "PolicyMatchCondition", + "doc" : "The matching condition in a filter criterion", + "symbols" : [ "EQUALS" ], + "symbolDocs" : { + "EQUALS" : "Whether the field matches the value" + } + }, + "doc" : "The condition for the criterion", + "default" : "EQUALS" + } ] + } + }, + "doc" : "A list of criteria to apply conjunctively (so all criteria must pass)" + } ] + }, + "doc" : "Filter to apply privileges to", + "optional" : true } ] }, "doc" : "The resource that the policy applies to. Not required for some 'Platform' privileges.", @@ -5334,7 +5380,7 @@ "doc" : "Additional properties", "optional" : true } ] - }, "com.linkedin.policy.DataHubActorFilter", "com.linkedin.policy.DataHubPolicyInfo", "com.linkedin.policy.DataHubResourceFilter", "com.linkedin.retention.DataHubRetentionConfig", "com.linkedin.retention.Retention", "com.linkedin.retention.TimeBasedRetention", "com.linkedin.retention.VersionBasedRetention", "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EditableSchemaFieldInfo", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeyConstraint", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey", "com.linkedin.tag.TagProperties" ], + }, "com.linkedin.policy.DataHubActorFilter", "com.linkedin.policy.DataHubPolicyInfo", "com.linkedin.policy.DataHubResourceFilter", "com.linkedin.policy.PolicyMatchCondition", "com.linkedin.policy.PolicyMatchCriterion", "com.linkedin.policy.PolicyMatchFilter", "com.linkedin.retention.DataHubRetentionConfig", "com.linkedin.retention.Retention", "com.linkedin.retention.TimeBasedRetention", "com.linkedin.retention.VersionBasedRetention", "com.linkedin.schema.ArrayType", "com.linkedin.schema.BinaryJsonSchema", "com.linkedin.schema.BooleanType", "com.linkedin.schema.BytesType", "com.linkedin.schema.DatasetFieldForeignKey", "com.linkedin.schema.DateType", "com.linkedin.schema.EditableSchemaFieldInfo", "com.linkedin.schema.EditableSchemaMetadata", "com.linkedin.schema.EnumType", "com.linkedin.schema.EspressoSchema", "com.linkedin.schema.FixedType", "com.linkedin.schema.ForeignKeyConstraint", "com.linkedin.schema.ForeignKeySpec", "com.linkedin.schema.KafkaSchema", "com.linkedin.schema.KeyValueSchema", "com.linkedin.schema.MapType", "com.linkedin.schema.MySqlDDL", "com.linkedin.schema.NullType", "com.linkedin.schema.NumberType", "com.linkedin.schema.OracleDDL", "com.linkedin.schema.OrcSchema", "com.linkedin.schema.OtherSchema", "com.linkedin.schema.PrestoDDL", "com.linkedin.schema.RecordType", "com.linkedin.schema.SchemaField", "com.linkedin.schema.SchemaFieldDataType", "com.linkedin.schema.SchemaMetadata", "com.linkedin.schema.SchemaMetadataKey", "com.linkedin.schema.Schemaless", "com.linkedin.schema.StringType", "com.linkedin.schema.TimeType", "com.linkedin.schema.UnionType", "com.linkedin.schema.UrnForeignKey", "com.linkedin.tag.TagProperties" ], "schema" : { "name" : "entities", "namespace" : "com.linkedin.entity", diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java index db2bdc1e0bcc40..ac7afa214cd299 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -34,6 +34,13 @@ public interface EntityClient { + @Nullable + public EntityResponse getV2( + @Nonnull String entityName, + @Nonnull final Urn urn, + @Nullable final Set aspectNames, + @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException; + @Nonnull @Deprecated public Entity get(@Nonnull final Urn urn, @Nonnull final Authentication authentication) diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java index da96418bfa76a0..c44e3429a77c72 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java @@ -66,6 +66,18 @@ public class JavaEntityClient implements EntityClient { private final TimeseriesAspectService _timeseriesAspectService; private final LineageSearchService _lineageSearchService; + @Nullable + public EntityResponse getV2( + @Nonnull String entityName, + @Nonnull final Urn urn, + @Nullable final Set aspectNames, + @Nonnull final Authentication authentication) throws RemoteInvocationException, URISyntaxException { + final Set projectedAspects = aspectNames == null + ? _entityService.getEntityAspectNames(entityName) + : aspectNames; + return _entityService.getEntityV2(entityName, urn, projectedAspects); + } + @Nonnull public Entity get(@Nonnull final Urn urn, @Nonnull final Authentication authentication) { return _entityService.getEntity(urn, ImmutableSet.of()); diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/OwnershipClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/OwnershipClient.java deleted file mode 100644 index 7d74858c9862cd..00000000000000 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/OwnershipClient.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.linkedin.entity.client; - -import com.linkedin.common.Ownership; -import com.linkedin.common.urn.Urn; -import com.linkedin.entity.EntityResponse; -import com.linkedin.r2.RemoteInvocationException; -import com.linkedin.restli.common.HttpStatus; -import com.linkedin.restli.server.RestLiServiceException; -import java.net.URISyntaxException; -import java.util.Collections; -import javax.annotation.Nullable; - -import static com.linkedin.metadata.Constants.*; - - -/** - * Basic client that fetches {@link Ownership} aspects from the Metadata Service. - */ -public class OwnershipClient { - - private final EntityClient _entityClient; - - public OwnershipClient(final EntityClient entityClient) { - _entityClient = entityClient; - } - - /** - * Retrieve the latest version of the standard {@link Ownership} aspect from the Metadata Service, - * using a raw {@link EntityClient}. - * - * @param urn stringified urn associated with the entity to fetch Ownership for. - * @return an instance of {@link Ownership} if one is found, or null if one is not found. - * @throws RemoteInvocationException if Rest.li throws an unexpected exception (aside from 404 not found) - */ - @Nullable - public Ownership getLatestOwnership(final String urn) throws RemoteInvocationException { - // Fetch the latest version of "ownership" aspect for the resource. - try { - Urn urnObj = Urn.createFromString(urn); - final EntityResponse entityResponse = _entityClient.batchGetV2( - urnObj.getEntityType(), - Collections.singleton(urnObj), - Collections.singleton(OWNERSHIP_ASPECT_NAME), - null).get(urnObj); - if (entityResponse != null && entityResponse.getAspects().containsKey(OWNERSHIP_ASPECT_NAME)) { - return new Ownership(entityResponse.getAspects().get(OWNERSHIP_ASPECT_NAME).getValue().data()); - } - return null; - } catch (RestLiServiceException e) { - if (HttpStatus.S_404_NOT_FOUND.equals(e.getStatus())) { - // No aspect exists. - return null; - } - throw e; - } catch (URISyntaxException ue) { - throw new RuntimeException(ue); - } - } -} diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index 375ad98e01f9ce..3c866e0bda0582 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -29,6 +29,7 @@ import com.linkedin.entity.EntitiesDoSetWritableRequestBuilder; import com.linkedin.entity.EntitiesRequestBuilders; import com.linkedin.entity.EntitiesV2BatchGetRequestBuilder; +import com.linkedin.entity.EntitiesV2GetRequestBuilder; import com.linkedin.entity.EntitiesV2RequestBuilders; import com.linkedin.entity.Entity; import com.linkedin.entity.EntityArray; @@ -65,7 +66,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.search.utils.QueryUtils.*; +import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; @Slf4j @@ -79,6 +80,16 @@ public RestliEntityClient(@Nonnull final Client restliClient) { super(restliClient); } + @Nullable + public EntityResponse getV2(@Nonnull String entityName, @Nonnull final Urn urn, + @Nullable final Set aspectNames, @Nonnull final Authentication authentication) + throws RemoteInvocationException, URISyntaxException { + final EntitiesV2GetRequestBuilder requestBuilder = ENTITIES_V2_REQUEST_BUILDERS.get() + .aspectsParam(aspectNames) + .id(urn.toString()); + return sendClientRequest(requestBuilder, authentication).getEntity(); + } + @Nonnull public Entity get(@Nonnull final Urn urn, @Nonnull final Authentication authentication) throws RemoteInvocationException { diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java index fcff0c01126e9f..5ba91ee2df2759 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityV2Resource.java @@ -1,7 +1,6 @@ package com.linkedin.metadata.resources.entity; import com.codahale.metrics.MetricRegistry; -import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.entity.EntityResponse; import com.linkedin.metadata.entity.EntityService; @@ -25,8 +24,8 @@ import javax.inject.Named; import lombok.extern.slf4j.Slf4j; -import static com.linkedin.metadata.resources.restli.RestliConstants.*; -import static com.linkedin.metadata.utils.PegasusUtils.*; +import static com.linkedin.metadata.resources.restli.RestliConstants.PARAM_ASPECTS; +import static com.linkedin.metadata.utils.PegasusUtils.urnToEntityName; /** @@ -54,14 +53,12 @@ public Task get(@Nonnull String urnStr, final String entityName = urnToEntityName(urn); final Set projectedAspects = aspectNames == null ? getAllAspectNames(entityName) : new HashSet<>(Arrays.asList(aspectNames)); - Map entities; try { - entities = _entityService.getEntitiesV2(entityName, ImmutableSet.of(urn), projectedAspects); + return _entityService.getEntityV2(entityName, urn, projectedAspects); } catch (Exception e) { throw new RuntimeException( String.format("Failed to get entity with urn: %s, aspects: %s", urn, projectedAspects), e); } - return entities.get(urn); }, MetricRegistry.name(this.getClass(), "get")); } diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index 0bfee57a04f886..c8073cb321e382 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -38,143 +38,30 @@ ] }, "privileges":[ - "EDIT_ENTITY" + "EDIT_ENTITY", + "VIEW_ENTITY_PAGE" ], - "displayName":"Root User - Edit All Datasets", - "resources":{ - "type":"dataset", - "allResources":true - }, - "description":"Grants full edit privileges for Datasets to root 'datahub' root user.", + "displayName":"Root User - Edit and View All Resources", + "description":"Grants full edit and view privileges for all resources to root 'datahub' root user.", "state":"ACTIVE", "type":"METADATA", "editable":false } }, { - "urn": "urn:li:dataHubPolicy:2", - "info": { - "actors":{ - "resourceOwners":false, - "allUsers":false, - "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] - }, - "privileges":[ - "EDIT_ENTITY" - ], - "displayName":"Root User - Edit All Dashboards", - "resources":{ - "type":"dashboard", - "allResources":true - }, - "description":"Grants full edit privileges for Dashboards to root datahub super user.", - "state":"ACTIVE", - "type":"METADATA", - "editable":false - } + "urn": "urn:li:dataHubPolicy:2" }, { - "urn": "urn:li:dataHubPolicy:3", - "info": { - "actors":{ - "resourceOwners":false, - "allUsers":false, - "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] - }, - "privileges":[ - "EDIT_ENTITY" - ], - "displayName":"Root User - Edit All Charts", - "resources":{ - "type":"chart", - "allResources":true - }, - "description":"Grants full edit privileges for Charts to root datahub super user.", - "state":"ACTIVE", - "type":"METADATA", - "editable":false - } + "urn": "urn:li:dataHubPolicy:3" }, { - "urn": "urn:li:dataHubPolicy:4", - "info": { - "actors":{ - "resourceOwners":false, - "allUsers":false, - "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] - }, - "privileges":[ - "EDIT_ENTITY" - ], - "displayName":"Root User - Edit All Tags", - "resources":{ - "type":"tag", - "allResources":true - }, - "description":"Grants full edit privileges for Tags to root datahub super user.", - "state":"ACTIVE", - "type":"METADATA", - "editable":false - } + "urn": "urn:li:dataHubPolicy:4" }, { - "urn": "urn:li:dataHubPolicy:5", - "info": { - "actors":{ - "resourceOwners":false, - "allUsers":false, - "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] - }, - "privileges":[ - "EDIT_ENTITY" - ], - "displayName":"Root User - Edit All Data Pipelines", - "resources":{ - "type":"dataFlow", - "allResources":true - }, - "description":"Grants full edit privileges for Data Pipelines to root datahub super user.", - "state":"ACTIVE", - "type":"METADATA", - "editable":false - } + "urn": "urn:li:dataHubPolicy:5" }, { - "urn": "urn:li:dataHubPolicy:6", - "info": { - "actors":{ - "resourceOwners":false, - "allUsers":false, - "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] - }, - "privileges":[ - "EDIT_ENTITY" - ], - "displayName":"Root User - Edit All Data Tasks", - "resources":{ - "type":"dataJob", - "allResources":true - }, - "description":"Grants full edit privileges for Data Tasks to root datahub super user.", - "state":"ACTIVE", - "type":"METADATA", - "editable":false - } + "urn": "urn:li:dataHubPolicy:6" }, { "urn": "urn:li:dataHubPolicy:7", @@ -201,78 +88,51 @@ } }, { - "urn": "urn:li:dataHubPolicy:8", - "info": { - "actors":{ - "resourceOwners":false, - "allUsers":false, - "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] - }, - "privileges":[ - "EDIT_ENTITY" - ], - "displayName":"Root User - Edit All Containers", - "resources":{ - "type":"container", - "allResources":true - }, - "description":"Grants full edit privileges for Containers to root 'datahub' root user.", - "state":"ACTIVE", - "type":"METADATA", - "editable":false - } + "urn": "urn:li:dataHubPolicy:8" + }, + { + "urn": "urn:li:dataHubPolicy:9" }, { - "urn": "urn:li:dataHubPolicy:9", + "urn": "urn:li:dataHubPolicy:10" + }, + { + "urn": "urn:li:dataHubPolicy:view-entity-page-all", "info": { "actors":{ "resourceOwners":false, - "allUsers":false, + "allUsers":true, "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] + "users":[] }, "privileges":[ - "EDIT_ENTITY" + "VIEW_ENTITY_PAGE" ], - "displayName":"Root User - Edit All Glossary Terms", - "resources":{ - "type":"glossaryTerm", - "allResources":true - }, - "description":"Grants full edit privileges for Glossary Terms to root 'datahub' root user.", + "displayName":"All Users - View Entity Page", + "description":"Grants entity view to all users", "state":"ACTIVE", "type":"METADATA", - "editable":false + "editable":true } }, { - "urn": "urn:li:dataHubPolicy:10", + "urn": "urn:li:dataHubPolicy:view-dataset-sensitive", "info": { "actors":{ "resourceOwners":false, - "allUsers":false, + "allUsers":true, "allGroups":false, - "users":[ - "urn:li:corpuser:datahub" - ] + "users":[] }, "privileges":[ - "EDIT_ENTITY" + "VIEW_DATASET_USAGE", + "VIEW_DATASET_PROFILE" ], - "displayName":"Root User - Edit All Groups", - "resources":{ - "type":"corpGroup", - "allResources":true - }, - "description":"Grants full edit privileges for Groups to root 'datahub' root user.", + "displayName":"All Users - View Dataset Sensitive Information", + "description":"Grants viewing privileges of usage and profile information of all datasets for all users", "state":"ACTIVE", "type":"METADATA", - "editable":false + "editable":true } } ] diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java index 05625991261fd6..db497825bd3cfa 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -80,6 +80,7 @@ public class Constants { public static final String SCHEMA_METADATA_ASPECT_NAME = "schemaMetadata"; public static final String EDITABLE_SCHEMA_METADATA_ASPECT_NAME = "editableSchemaMetadata"; public static final String VIEW_PROPERTIES_ASPECT_NAME = "viewProperties"; + public static final String DATASET_PROFILE_ASPECT_NAME = "datasetProfile"; // Chart public static final String CHART_KEY_ASPECT_NAME = "chartKey"; diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index e916890e1d4ed9..858c19f288ed1d 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -70,6 +70,11 @@ public class PoliciesConfig { // Resource Privileges // + public static final Privilege VIEW_ENTITY_PAGE_PRIVILEGE = Privilege.of( + "VIEW_ENTITY_PAGE", + "View Entity Page", + "The ability to view the entity page."); + public static final Privilege EDIT_ENTITY_TAGS_PRIVILEGE = Privilege.of( "EDIT_ENTITY_TAGS", "Edit Tags", @@ -121,6 +126,7 @@ public class PoliciesConfig { "The ability to edit any information about an entity. Super user privileges."); public static final List COMMON_ENTITY_PRIVILEGES = ImmutableList.of( + VIEW_ENTITY_PAGE_PRIVILEGE, EDIT_ENTITY_TAGS_PRIVILEGE, EDIT_ENTITY_GLOSSARY_TERMS_PRIVILEGE, EDIT_ENTITY_OWNERS_PRIVILEGE, @@ -151,6 +157,16 @@ public class PoliciesConfig { "The ability to edit the column (field) descriptions associated with a dataset schema." ); + public static final Privilege VIEW_DATASET_USAGE_PRIVILEGE = Privilege.of( + "VIEW_DATASET_USAGE", + "View Dataset Usage", + "The ability to access dataset usage information (includes usage statistics and queries)."); + + public static final Privilege VIEW_DATASET_PROFILE_PRIVILEGE = Privilege.of( + "VIEW_DATASET_PROFILE", + "View Dataset Profile", + "The ability to access dataset profile (snapshot statistics)"); + // Tag Privileges public static final Privilege EDIT_TAG_COLOR_PRIVILEGE = Privilege.of( "EDIT_TAG_COLOR", @@ -181,6 +197,8 @@ public class PoliciesConfig { "Datasets indexed by DataHub", Stream.of( COMMON_ENTITY_PRIVILEGES, ImmutableList.of( + VIEW_DATASET_USAGE_PRIVILEGE, + VIEW_DATASET_PROFILE_PRIVILEGE, EDIT_DATASET_COL_DESCRIPTION_PRIVILEGE, EDIT_DATASET_COL_TAGS_PRIVILEGE, EDIT_DATASET_COL_GLOSSARY_TERMS_PRIVILEGE, @@ -290,7 +308,7 @@ public class PoliciesConfig { EDIT_ENTITY_PRIVILEGE) ); - public static final List RESOURCE_PRIVILEGES = ImmutableList.of( + public static final List ENTITY_RESOURCE_PRIVILEGES = ImmutableList.of( DATASET_PRIVILEGES, DASHBOARD_PRIVILEGES, CHART_PRIVILEGES, @@ -305,6 +323,20 @@ public class PoliciesConfig { NOTEBOOK_PRIVILEGES ); + // Merge all entity specific resource privileges to create a superset of all resource privileges + public static final ResourcePrivileges ALL_RESOURCE_PRIVILEGES = ResourcePrivileges.of( + "all", + "All Types", + "All Types", + ENTITY_RESOURCE_PRIVILEGES.stream().flatMap(resourcePrivileges -> resourcePrivileges.getPrivileges().stream()).distinct().collect( + Collectors.toList()) + ); + + public static final List RESOURCE_PRIVILEGES = + ImmutableList.builder().addAll(ENTITY_RESOURCE_PRIVILEGES) + .add(ALL_RESOURCE_PRIVILEGES) + .build(); + @Data @Getter @AllArgsConstructor