feature!: Squashed commit of features and fixes. (#53)

* Stability improvements: Redact_folder: Reduce checks for completion to every 10s. Limit parallel uploads and cache connections. Retry requests that fail due to connection reasons. Fix premature closing of http client instance. Stream download to file instead of to memory. * Add option for automated deletion of input file to redact-folder. --auto_delete_input_file deletes the input file after successful processing * Add verbose logging option to log more details. --verbose_logging switch adds datetime to log and sets level to debug. * Update dependencies to fix vulnerabilities, this also required dropping Python 3.6 support. * Redact_file: Provide additional config options. * feature: Fix test-integration to run on mac and windows. BREAKING CHANGE: Drop Python 3.6 support.
brighter-ai · Aug 10, 2022 · efd2b5f · efd2b5f
1 parent 2b965c4
commit efd2b5f
Show file tree

Hide file tree

Showing 20 changed files with 351 additions and 110 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        python-version: [ 3.7, 3.8, 3.9 ]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9 ]
+        python-version: [ 3.7, 3.8, 3.9 ]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

diff --git a/.gitignore b/.gitignore
@@ -109,6 +109,7 @@ MANIFEST
 
 # VS Code
 *.vscode
+*.code-workspace
 
 # Installer logs
 pip-log.txt
@@ -220,3 +221,6 @@ cython_debug/
 # ignore pipenv stuff until we support it
 Pipfile
 Pipfile.lock
+
+.DS_Store
+
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-VERSION=4.8.1
+VERSION=5.0.0
 
 SHELL := /bin/bash
 

diff --git a/README.md b/README.md
@@ -2,24 +2,23 @@
 
 # Redact Python Client
 
-[![Unit + Integration Tests](https://github.com/brighter-ai/redact-client/actions/workflows/pytest.yml/badge.svg)](https://github.com/brighter-ai/redact-client/actions/workflows/pytest.yml)   [![Python Linter](https://github.com/brighter-ai/redact-client/actions/workflows/flake8.yml/badge.svg)](https://github.com/brighter-ai/redact-client/actions/workflows/flake8.yml)  [![Python Package Build](https://github.com/brighter-ai/redact-client/actions/workflows/build.yml/badge.svg)](https://github.com/brighter-ai/redact-client/actions/workflows/build.yml)
-
+[![Unit + Integration Tests](https://github.com/brighter-ai/redact-client/actions/workflows/pytest.yml/badge.svg)](https://github.com/brighter-ai/redact-client/actions/workflows/pytest.yml) [![Python Linter](https://github.com/brighter-ai/redact-client/actions/workflows/flake8.yml/badge.svg)](https://github.com/brighter-ai/redact-client/actions/workflows/flake8.yml) [![Python Package Build](https://github.com/brighter-ai/redact-client/actions/workflows/build.yml/badge.svg)](https://github.com/brighter-ai/redact-client/actions/workflows/build.yml)
 
 - [Overview](#overview)
 - [Installation](#installation)
 - [Quickstart](#quickstart)
-    + [Examples](#examples)
+  - [Examples](#examples)
 - [Library Usage](#library-usage)
-    + [Batch Processing](#batch-file-processing)
-    + [API Requests](#api-requests)
-    + [Redact Jobs](#redact-jobs)
-
+  - [Batch Processing](#batch-file-processing)
+  - [API Requests](#api-requests)
+  - [Redact Jobs](#redact-jobs)
 
 # Overview
 
 This repository provides access to [brighter AI's](https://brighter.ai/) [Redact API](https://docs.brighter.ai/docs/) for the anonymization of faces and license plates.
 
 Learn more:
+
 - About us: [Brighter AI](https://brighter.ai/)
 - Try [Redact](https://brighter.ai/product/) online
 - [API Documentation](https://docs.brighter.ai/docs/)
@@ -34,7 +33,6 @@ pip install git+https://github.com/brighter-ai/redact-client.git
 
 For a specific version, append `@[version]`.
 
-
 ## Quickstart
 
 The pip package automatically installs two command-line shortcuts (`redact_file` and `redact_folder`) that let you
@@ -55,7 +53,6 @@ Usage: redact_folder [OPTIONS] IN_DIR OUT_DIR
 
 Add `--help` to see additional options.
 
-
 ### Examples
 
 Anonymize an individual image from the command line:
@@ -64,7 +61,8 @@ Anonymize an individual image from the command line:
 redact_file image.jpg images blur --redact-url=http://127.0.0.1:8787
 ```
 
-Per default, the result will be stored in `image_redacted.jpg`.
+Per default, the result will be stored in `image_redacted.jpg` if the OUT_DIR
+is the same as IN_DIR, and the original file name is used if OUT_DIR != IN_DIR.
 
 Larger amounts of data (images in this case) can be
 anonymized in batches:
@@ -73,7 +71,6 @@ anonymized in batches:
 redact_folder ./in_dir ./out_dir images images blur --redact-url=127.0.0.1:8787
 ```
 
-
 ## Library Usage
 
 The `redact` package itself provides different ways to use the Redact API from Python.

diff --git a/pyproject.toml b/pyproject.toml
@@ -11,11 +11,10 @@ module = "redact"
 author = "brighter AI"
 author-email = "[email protected]"
 home-page = "https://github.com/brighter-ai"
-requires-python = ">=3.6"
+requires-python = ">=3.7"
 requires = [
-    "httpcore~=0.12.3",
-    "httpx~=0.17.1",
-    "pydantic~=1.8.2",
-    "tqdm~=4.60.0",
-    "typer~=0.3.2"
+  "httpx~=0.23.0",
+  "pydantic~=1.9.1",
+  "tqdm~=4.64.0",
+  "typer~=0.6.1",
 ]
diff --git a/redact/__init__.py b/redact/__init__.py
@@ -2,7 +2,7 @@
 Python client for "brighter Redact"
 """
 
-__version__ = "4.8.1"
+__version__ = "5.0.0"
 
 from .redact_instance import RedactInstance  # noqa
 from .redact_job import RedactJob  # noqa

diff --git a/redact/__main__.py b/redact/__main__.py
@@ -1,4 +1,5 @@
 from typing import Optional
+import logging
 
 import typer
 
@@ -162,6 +163,12 @@ def redact_folder(
     auto_delete_job: bool = typer.Option(
         True, help="Specify whether to automatically delete the job from the backend"
     ),
+    auto_delete_input_file: bool = typer.Option(
+        False,
+        help="Specify whether to automatically delete the input file "
+        "from the input folder after processing of a file completed.",
+    ),
+    verbose_logging: bool = typer.Option(False, help="Enable very noisy logging."),
 ):
     job_args = JobArguments(
         region=region,
@@ -174,6 +181,13 @@ def redact_folder(
         face_determination_threshold=face_determination_threshold,
     )
 
+    if verbose_logging:
+        logging.basicConfig(
+            format="%(asctime)s %(levelname)s -- %(message)s", level=logging.DEBUG
+        )
+    else:
+        logging.basicConfig(level=settings.log_level)
+
     rdct_folder(
         in_dir=in_dir,
         out_dir=out_dir,
@@ -189,6 +203,7 @@ def redact_folder(
         ignore_warnings=ignore_warnings,
         skip_existing=skip_existing,
         auto_delete_job=auto_delete_job,
+        auto_delete_input_file=auto_delete_input_file,
     )
 
 

diff --git a/redact/redact_instance.py b/redact/redact_instance.py
@@ -1,4 +1,4 @@
-from typing import Optional, IO, Union
+from typing import Optional, BinaryIO, IO, Union
 
 from redact.data_models import ServiceType, OutputType, JobArguments, JobLabels
 from redact.redact_job import RedactJob
@@ -21,6 +21,7 @@ def __init__(
     ):
         """
         The default way for creating RedactInstance objects is through RedactInstance.create().
+        Here you can provide your own RedactRequests instance.
         """
         self.redact_requests = redact_requests
         self.service = service
@@ -45,9 +46,9 @@ def create(
 
     def start_job(
         self,
-        file: IO,
+        file: BinaryIO,
         job_args: Optional[JobArguments] = None,
-        licence_plate_custom_stamp: Optional[IO] = None,
+        licence_plate_custom_stamp: Optional[BinaryIO] = None,
         custom_labels: Optional[Union[str, IO, JobLabels]] = None,
     ) -> RedactJob:
         post_response = self.redact_requests.post_job(
@@ -58,7 +59,6 @@ def start_job(
             licence_plate_custom_stamp=licence_plate_custom_stamp,
             custom_labels=custom_labels,
         )
-
         return RedactJob(
             redact_requests=self.redact_requests,
             service=self.service,

diff --git a/redact/redact_job.py b/redact/redact_job.py
@@ -1,5 +1,5 @@
 import time
-
+from pathlib import Path
 from uuid import UUID
 
 from redact.data_models import ServiceType, OutputType, JobStatus, JobResult, JobLabels
@@ -51,6 +51,15 @@ def download_result(self, ignore_warnings: bool = False) -> JobResult:
             ignore_warnings=ignore_warnings,
         )
 
+    def download_result_to_file(self, file: Path, ignore_warnings: bool = False):
+        self.redact.write_output_to_file(
+            service=self.service,
+            out_type=self.out_type,
+            output_id=self.output_id,
+            file=file,
+            ignore_warnings=ignore_warnings,
+        )
+
     def delete(self):
         return self.redact.delete_output(
             service=self.service, out_type=self.out_type, output_id=self.output_id