diff --git a/.travis.yml b/.travis.yml
index 00a03ed..a5d8599 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,9 +7,7 @@ python:
install:
- "pip install -U pip flake8 pep257 pytest-cov codecov codacy-coverage pluggy"
- - "python setup.py build"
- - "python setup.py bdist --formats=zip"
- - "python setup.py install"
+ - "pip install ."
script:
- "flake8 ./fido"
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..a31bc1c
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,33 @@
+FROM python:3.6-alpine as builder
+
+LABEL maintainer="carl.wilson@openpreservation.org" \
+ org.openpreservation.vendor="Open Preservation Foundation" \
+ version="0.1"
+
+RUN apk update && apk --no-cache --update-cache add gcc build-base libxml2-dev libxslt-dev git
+
+WORKDIR /src
+
+COPY setup.py setup.py
+COPY requirements.txt requirements.txt
+COPY README.md README.md
+COPY fido/* fido/
+
+RUN mkdir /install && pip install -U pip && pip install -r requirements.txt --prefix=/install && pip install --prefix=/install .
+
+FROM python:3.6-alpine
+
+RUN apk update && apk add --no-cache --update-cache libc6-compat libstdc++ bash libxslt
+RUN install -d -o root -g root -m 755 /opt && adduser --uid 1000 -h /opt/fido_sigs -S eark && pip install -U pip python-dateutil
+
+WORKDIR /opt/fido_sigs
+
+COPY --from=builder /install /usr/local
+COPY . /opt/fido_sigs/
+RUN chown -R eark:users /opt/fido_sigs
+
+USER eark
+
+EXPOSE 5000
+ENV FLASK_APP='fido.signatures'
+ENTRYPOINT flask run --host "0.0.0.0" --port "5000"
diff --git a/README.md b/README.md
index bf93a97..80f90a8 100644
--- a/README.md
+++ b/README.md
@@ -9,31 +9,34 @@ FIDO is a command-line tool to identify the file formats of digital objects.
It is designed for simple integration into automated work-flows.
FIDO uses the UK National Archives (TNA) PRONOM File Format and Container descriptions.
-PRONOM is available from http://www.nationalarchives.gov.uk/pronom/
+PRONOM is available from
See [LICENSE](LICENSE.txt) for license information.
-* Download from: https://github.com/openpreserve/fido/releases
-* Usage guide: http://wiki.opf-labs.org/display/KB/FIDO+usage+guide
+* Download from:
+* Usage guide:
* Author: Adam Farquhar (BL), 2010
* Maintainer: Maurice de Rooij (OPF/NANETH), 2011, 2012, 2013, Misty de Meo 2014, 2015, 2016, Holly Becker 2016
Usage
-----
-```
-usage: fido.py [-h] [-v] [-q] [-recurse] [-zip] [-nocontainer] [-pronom_only]
- [-input INPUT] [-filename FILENAME] [-useformats INCLUDEPUIDS]
- [-nouseformats EXCLUDEPUIDS] [-matchprintf FORMATSTRING]
- [-nomatchprintf FORMATSTRING] [-bufsize BUFSIZE]
- [-container_bufsize CONTAINER_BUFSIZE]
- [-loadformats XML1,...,XMLn] [-confdir CONFDIR]
- [FILE [FILE ...]]
+```shell
+usage: fido [-h] [-v] [-q] [-recurse] [-zip] [-noextension] [-nocontainer]
+ [-pronom_only] [-input INPUT] [-filename FILENAME]
+ [-useformats INCLUDEPUIDS] [-nouseformats EXCLUDEPUIDS]
+ [-matchprintf FORMATSTRING] [-nomatchprintf FORMATSTRING]
+ [-bufsize BUFSIZE] [-sigs SIG_ACT]
+ [-container_bufsize CONTAINER_BUFSIZE]
+ [-loadformats XML1,...,XMLn] [-confdir CONFDIR]
+ [FILE [FILE ...]]
```
positional arguments:
+
* `FILE`: files to check. If the file is -, then read content from stdin. In this case, python must be invoked with `-u` or it may convert the line terminators.
optional arguments:
+
* `-h`, `--help`: show this help message and exit
* `-v`: show version information
* `-q`: run (more) quietly
@@ -48,6 +51,10 @@ optional arguments:
* `-matchprintf FORMATSTRING`: format string (Python style) to use on match. See nomatchprintf, README.txt.
* `-nomatchprintf FORMATSTRING`: format string (Python style) to use if no match. See README.txt
* `-bufsize BUFSIZE`: size (in bytes) of the buffer to match against (default=131072 bytes)
+* `-sigs SIG_ACT`: SIG_ACT "check" for new version of signature file for download.
+ SIG_ACT "list" list all available sig file versions.
+ SIG_ACT "update" to automatically update to latest available sig file.
+ SIG_ACT "n" download and use version n.
* `-container_bufsize CONTAINER_BUFSIZE`: size (in bytes) of the buffer to match against (default=524288 bytes)
* `-loadformats XML1,...,XMLn`: comma separated string of XML format files to add.
* `-confdir CONFDIR`: configuration directory to load_fido_xml, for example, the format specifications from.
@@ -55,11 +62,11 @@ optional arguments:
Installation
------------
-(also see: http://wiki.opf-labs.org/display/KB/FIDO+usage+guide)
+(also see: )
Any platform
-1. Download the latest zip release from https://github.com/openpreserve/fido/releases
+1. Download the latest zip release from
2. Unzip into some directory
3. Open a command shell, cd to the directory that you placed the zip contents into
4. Run `python setup.py install` to install FIDO and dependencies. This may require sudo on Linux/OSX or admin privileges on Windows.
@@ -75,11 +82,42 @@ Using pip
Updating signatures
-------------------
-To update FIDO with the latest PRONOM file format definitions, run:
- `fido-update-signatures`
-This is an interactive CLI script which downloads the latest PRONOM signature file and signatures. Please note that it can take a while to download all PUID signatures.
+Signatures can be updated from the OPF's signature service.
+The service is pull only and iit's location is in the `versions.xml`
+configuration file as
+
+```xml
+https://fidosigs.openpreservation.org
+```
+
+To check what version of the PRONOM signatures you are using
+type: `fido -v` and you'll see something like:
-If you are having trouble running the script due to firewall restrictions, see OPF wiki: http://wiki.opf-labs.org/display/PT/Command+Line+Interface+proxy+usage
+```shell
+FIDO v1.6.0 (pronom-xml-95.zip, container-signature-20200121.xml, format_extensions.xml)
+```
+
+Here `pronom-xml-95.zip` denotes PRONOM version 95. To see if a more recent
+set of signatures is available type `fido -sigs check` which will report back:
+
+```shell
+Updated signatures v104 are available, current version is v95
+```
+
+if new signatures are available or
+
+```shell
+Your signature files are up to date, current version is v104
+```
+
+if not. To update signatures to the latest version type `fido -sigs update`:
+
+```shell
+Updated signatures v104 are available, current version is v95
+Updating signatures
+```
+
+If you are having trouble due to firewall restrictions, see OPF wiki:
Please note that this WILL NOT update the container signature file located in the 'conf' folder.
The reason for this that the PRONOM container signature file contains special types
@@ -97,6 +135,8 @@ or a pip installation will handle dependencies.
FIDO 1.3.3 and later have experimental Python 3 support.
+FIDO 1.4 and later have Python 3 support.
+
Format Definitions
------------------
@@ -118,11 +158,12 @@ an object called info with the following fields:
* `printnomatch`: `info.count` (file N)
The defaults for FIDO 1.0 are:
+
* `printmatch`:
- * `"OK,%(info.time)s,%(info.puid)s,%(info.formatname)s,%(info.signaturename)s,%(info.filesize)s,\"%(info.filename)s\",\"%(info.mimetype)s\",\"%(info.matchtype)s\"\n"`
+* `"OK,%(info.time)s,%(info.puid)s,%(info.formatname)s,%(info.signaturename)s,%(info.filesize)s,\"%(info.filename)s\",\"%(info.mimetype)s\",\"%(info.matchtype)s\"\n"`
* `printnomatch`:
- * `"KO,%(info.time)s,,,,%(info.filesize)s,\"%(info.filename)s\",,\"%(info.matchtype)s\"\n"`
+* `"KO,%(info.time)s,,,,%(info.filesize)s,\"%(info.filename)s\",,\"%(info.matchtype)s\"\n"`
It can be useful to provide an empty string for either, for example to ignore all failed matches, or all successful ones (see examples below).
Note that a newline needs to be added to the end of the string using \n.
@@ -131,10 +172,11 @@ Matchtypes
-----------
FIDO returns the following matchtypes:
-- fail: the object could not be identified with signature or file extension
-- extension: the object could only be identified by file extension
-- signature: the object has been identified with (a) PRONOM signature(s)
-- container: the object has been idenfified with (a) PRONOM container signature(s)
+
+* fail: the object could not be identified with signature or file extension
+* extension: the object could only be identified by file extension
+* signature: the object has been identified with (a) PRONOM signature(s)
+* container: the object has been idenfified with (a) PRONOM container signature(s)
In some cases multiple results are returned.
@@ -152,14 +194,14 @@ Take input from a list of files:
Linux:
-```
+```shell
ls > files.txt
python fido.py -input files.txt
```
Windows:
-```
+```shell
dir /b > files.txt
python fido.py -input files.txt
```
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index b8eac3c..83ef434 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -8,8 +8,38 @@ Copyright 2010 The Open Preservation Foundation
Fido is made available under the Apache License, Version 2.0; see the file
LICENSE.txt for details.
+Fido 1.6.0rc1
+-------------
+
+2022-03-29
+
+New command line options for updating signatures, see
+
+- PRONOM signatures can now be updated from a web service [[#202][]].
+- PRONOM v104 support with successful signature compilation (see issue [#203][]) [[#204][]].
+- Closed issue [#100][], Added Unicode support for Windows Python 2.7 [[#200][]].
+- Generated signature file now validated against XSD schema [[#197][]].
+- Refactoring and cleared final PEP and FLAKE code lint warnings [[#197][]].
+- Closed issue [#150][], trapped some of the signature compliation issues [[#197][]].
+- Closed issue [#179][], [#198][]: Crash on XLS format by updating olefile version to 0.46 [[#195][]].
+- Closed issue [#179][]: Crash on XLS format by updating olefile version to 0.46 [[#195][]].
+- Closed issue [#192][]: Fixed signature file defaults [[#193][]].
+
+[#100]: https://github.com/openpreserve/fido/issues/100
+[#150]: https://github.com/openpreserve/fido/issues/150
+[#179]: https://github.com/openpreserve/fido/issues/179
+[#192]: https://github.com/openpreserve/fido/issues/192
+[#193]: https://github.com/openpreserve/fido/pull/193
+[#195]: https://github.com/openpreserve/fido/pull/195
+[#198]: https://github.com/openpreserve/fido/issues/198
+[#200]: https://github.com/openpreserve/fido/pull/200
+[#202]: https://github.com/openpreserve/fido/pull/202
+[#203]: https://github.com/openpreserve/fido/issues/203
+[#204]: https://github.com/openpreserve/fido/pull/204
+
Fido 1.4.0
-------------
+
2018-12-19
- Python 3 support [[#156][]]
diff --git a/fido/conf/versions.xml b/fido/conf/versions.xml
index 98aa166..3c60cb3 100644
--- a/fido/conf/versions.xml
+++ b/fido/conf/versions.xml
@@ -1,8 +1,9 @@
- 96
- formats-v96.xml
+ 104
+ pronom-xml-104.zip
container-signature-20200121.xml
format_extensions.xml
- 1.4.1
+ 1.6.0rc1
+ https://fidosigs.openpreservation.org
\ No newline at end of file
diff --git a/fido/fido.py b/fido/fido.py
index 520d59c..51fbfee 100755
--- a/fido/fido.py
+++ b/fido/fido.py
@@ -31,7 +31,7 @@
from fido import __version__, CONFIG_DIR
from fido.package import OlePackage, ZipPackage
-from fido.versions import get_local_versions
+from fido.versions import get_local_versions, sig_file_actions
from fido.char_handler import escape
@@ -64,6 +64,7 @@
class PerfTimer:
"""Utility class that carries out simple process timings."""
+
def __init__(self):
"""New instance with start time running."""
self.start_time = perf_counter()
@@ -78,7 +79,10 @@ def duration(self):
class Fido:
+ """Main FIDO application class."""
+
def __init__(self, quiet=False, bufsize=None, container_bufsize=None, printnomatch=None, printmatch=None, zip=False, nocontainer=False, handle_matches=None, conf_dir=CONFIG_DIR, format_files=None, containersignature_file=None):
+ """Initialise a FIDO class instance."""
global defaults
self.quiet = quiet
self.bufsize = defaults['bufsize'] if bufsize is None else bufsize
@@ -107,10 +111,7 @@ def __init__(self, quiet=False, bufsize=None, container_bufsize=None, printnomat
self.externalsig = ET.XML('External')
def convert_container_sequence(self, sig):
- """
- Parse the PRONOM container sequences and convert to regular
- expressions.
- """
+ """Parse the PRONOM container sequences and convert to regular expressions."""
# The sequence is regex matching bytes from a file so the sequence must also be bytes
seq = b'(?s)'
inq = False
@@ -210,6 +211,7 @@ def format_signature_attributes(element):
return signatures
def match_container(self, signature_type, klass, file, signature_file):
+ """Return the signature matches for a container."""
puids = klass(file, self.extract_signatures(signature_file, signature_type=signature_type)).detect_formats()
results = []
for puid in puids:
@@ -221,6 +223,7 @@ def match_container(self, signature_type, klass, file, signature_file):
def load_fido_xml(self, file):
"""
Load the fido format information from @param file.
+
As a side-effect, set self.formats.
@return list of ElementTree.Element, one for each format.
"""
@@ -250,30 +253,38 @@ def process_format_element(self, element):
# To delete a format: (1) remove from self.formats, (2) remove from puid_format_map, (3) remove from selt.puid_has_priority_over_map
def get_signatures(self, format):
+ """Return the signatures for the format element."""
return format.findall('signature')
def has_priority_over(self, format, possibly_inferior):
+ """Return true if format has priority over possibly inferior."""
return self.get_puid(possibly_inferior) in self.puid_has_priority_over_map[self.get_puid(format)]
def get_puid(self, format):
+ """Return the PUID for the format."""
return format.find('puid').text
def get_patterns(self, signature):
+ """Return the patterns for a signature."""
return signature.findall('pattern')
def get_pos(self, pat):
+ """Return the position from a pattern."""
return pat.find('position').text
def get_regex(self, pat):
+ """Return the UTF-8 encoded regex from a pattern."""
# The regex is matching bytes from a file so regex must also be bytes
return pat.find('regex').text.encode('utf8')
def get_extension(self, format):
+ """Return the extension for a format."""
return format.find('extension').text
def print_matches(self, fullname, matches, delta_t, matchtype=''):
"""
The default match handler. Prints out information for each match in the list.
+
@param fullname is name of the file being matched
@param matches is a list of (format, signature)
@param delta_t is the time taken for the match.
@@ -330,9 +341,7 @@ class Info:
})
def print_summary(self, secs):
- """
- Print summary information on the number of matches and time taken.
- """
+ """Print summary information on the number of matches and time taken."""
count = self.current_count
if not self.quiet:
rate = (int(round(count / secs)) if secs != 0 else 9999)
@@ -342,6 +351,7 @@ def print_summary(self, secs):
def identify_file(self, filename, extension=True):
"""
Identify the type of @param filename.
+
Call self.handle_matches instead of returning a value.
"""
self.current_file = filename
@@ -387,8 +397,9 @@ def identify_file(self, filename, extension=True):
def identify_contents(self, filename, fileobj=None, type=False, extension=True):
"""
- Identify each item in a container (such as a zip or tar file). Call
- self.handle_matches on each item.
+ Identify each item in a container (such as a zip or tar file).
+
+ Call self.handle_matches on each item.
@param fileobj could be a file, or a stream.
"""
if not type:
@@ -411,7 +422,8 @@ def identify_contents(self, filename, fileobj=None, type=False, extension=True):
def identify_multi_object_stream(self, stream, extension=True):
"""
- Does not work!
+ Method does not work.
+
Stream may contain one or more objects each with an HTTP style header
that must include content-length. The headers consist of keyword:value
pairs terminated by a newline. There must be a newline following the
@@ -448,6 +460,7 @@ def identify_multi_object_stream(self, stream, extension=True):
def identify_stream(self, stream, filename, extension=True):
"""
Identify the type of @param stream.
+
Call self.handle_matches instead of returning a value.
Does not close stream.
"""
@@ -481,6 +494,8 @@ def identify_stream(self, stream, filename, extension=True):
def container_type(self, matches):
"""
+ Return true if this is a container type.
+
Determine if one of the @param matches is the format of a container
that we can look inside of (e.g., zip, tar).
@return False, zip, or tar.
@@ -499,6 +514,8 @@ def container_type(self, matches):
def can_recurse_into_container(self, container_type):
"""
+ Return true if it is possible to recursively process this container.
+
Determine if the passed container type can:
a) be extracted, and
b) contain individual files which can be identified separately.
@@ -510,6 +527,7 @@ def can_recurse_into_container(self, container_type):
return container_type in ('zip', 'tar')
def blocking_read(self, file, bytes_to_read):
+ """Perform a blocking read and return the buffer."""
bytes_read = 0
buffer = b''
while bytes_read < bytes_to_read:
@@ -523,8 +541,9 @@ def blocking_read(self, file, bytes_to_read):
def get_buffers(self, stream, length=None, seekable=False):
"""
- Return buffers from the beginning and end of stream and the number of
- bytes read if there may be more bytes in the stream.
+ Return buffers from the beginning and end of stream.
+
+ Includes number of bytes read if there may be more bytes in the stream.
If length is None, return the length as found.
If seekable is False, the steam does not support a seek operation.
@@ -570,7 +589,8 @@ def get_buffers(self, stream, length=None, seekable=False):
def walk_zip(self, filename, fileobj=None, extension=True):
"""
- Identify the type of each item in the zip
+ Identify the type of each item in the zip.
+
@param fileobj. If fileobj is not provided, open.
@param filename.
Call self.handle_matches instead of returning a value.
@@ -608,6 +628,7 @@ def walk_zip(self, filename, fileobj=None, extension=True):
def walk_tar(self, filename, fileobj, extension=True):
"""
Identify the type of each item in the tar.
+
@param fileobj. If fileobj is not provided, open.
@param filename.
Call self.handle_matches instead of returning a value.
@@ -634,6 +655,7 @@ def walk_tar(self, filename, fileobj, extension=True):
def as_good_as_any(self, f1, match_list):
"""
Return True if the proposed format is as good as any in the match_list.
+
For example, if there is no format in the match_list that has priority over the proposed one
"""
if match_list != []:
@@ -646,9 +668,7 @@ def as_good_as_any(self, f1, match_list):
return True
def buffered_read(self, file_pos, overlap):
- """
- Buffered read of data chunks.
- """
+ """Buffered read of data chunks."""
buf = ""
if not overlap:
bufsize = self.container_bufsize
@@ -667,6 +687,7 @@ def buffered_read(self, file_pos, overlap):
def match_formats(self, bofbuffer, eofbuffer):
"""
Apply the patterns for formats to the supplied buffers.
+
@return a match list of (format, signature) tuples.
The list has inferior matches removed.
"""
@@ -713,9 +734,7 @@ def match_formats(self, bofbuffer, eofbuffer):
return result
def match_extensions(self, filename):
- """
- Return the list of (format, self.externalsig) for every format whose extension matches the filename.
- """
+ """Return the list of (format, self.externalsig) for every format whose extension matches the filename."""
myext = os.path.splitext(filename)[1].lower().lstrip(".")
result = []
if not myext:
@@ -729,6 +748,7 @@ def match_extensions(self, filename):
return result
def copy_stream(self, source, target):
+ """Copy the stream from source to target."""
while True:
buf = source.read(self.bufsize)
if len(buf) == 0:
@@ -737,9 +757,7 @@ def copy_stream(self, source, target):
def list_files(roots, recurse=False):
- """
- Return the files one at a time. Roots could be a fileobj or a list.
- """
+ """Return the files one at a time. Roots could be a fileobj or a list."""
for root in roots:
root = (root if root[-1] != '\n' else root[:-1])
root = os.path.normpath(root)
@@ -761,8 +779,8 @@ def set_up_platform():
def main(args=None):
+ """Main FIDO method."""
set_up_platform()
-
if not args:
args = sys.argv[1:]
@@ -785,6 +803,7 @@ def main(args=None):
parser.add_argument('-matchprintf', metavar='FORMATSTRING', default=None, help='format string (Python style) to use on match. See nomatchprintf, README.txt.')
parser.add_argument('-nomatchprintf', metavar='FORMATSTRING', default=None, help='format string (Python style) to use if no match. See README.txt')
parser.add_argument('-bufsize', type=int, default=None, help='size (in bytes) of the buffer to match against (default=' + str(defaults['bufsize']) + ' bytes)')
+ parser.add_argument('-sigs', default=None, metavar='SIG_ACT', help='SIG_ACT "check" for new version\nSIG_ACT "update" to latest\nSIG_ACT "list" available versions\nSIG_ACT "n" use version n.')
parser.add_argument('-container_bufsize', type=int, default=None, help='size (in bytes) of the buffer to match against (default=' + str(defaults['container_bufsize']) + ' bytes)')
parser.add_argument('-loadformats', default=None, metavar='XML1,...,XMLn', help='comma separated string of XML format files to add.')
parser.add_argument('-confdir', default=CONFIG_DIR, help='configuration directory to load_fido_xml, for example, the format specifications from.')
@@ -813,6 +832,10 @@ def main(args=None):
sys.stdout.write(versionHeader)
sys.exit(0)
+ if args.sigs:
+ sig_file_actions(args.sigs.lower())
+ sys.exit(0)
+
if args.matchprintf:
try:
args.matchprintf = args.matchprintf.decode('string_escape')
diff --git a/fido/versions.py b/fido/versions.py
index 83be939..96c0566 100644
--- a/fido/versions.py
+++ b/fido/versions.py
@@ -20,11 +20,14 @@
from __future__ import absolute_import
import os
+import re
+import importlib_resources
+import sys
+import requests
+import six
from xml.etree import ElementTree as ET
from xml.etree.ElementTree import parse, ParseError
-import six
-
from fido import CONFIG_DIR
@@ -41,6 +44,7 @@ class LocalVersions(object):
container-signature-20160121.xml
format_extensions.xml
1.2.2
+ https://fidosigs.openpreservation.org
"""
@@ -50,6 +54,7 @@ class LocalVersions(object):
'pronom_container_signature': 'pronomContainerSignature',
'fido_extension_signature': 'fidoExtensionSignature',
'update_script': 'updateScript',
+ 'update_site': 'updateSite',
}
ROOT_ELEMENT = 'versions'
@@ -101,3 +106,67 @@ def write(self):
def get_local_versions(config_dir=CONFIG_DIR):
"""Return an instance of LocalVersions loaded with `conf/versions.xml`."""
return LocalVersions(os.path.join(config_dir, 'versions.xml'))
+
+
+def sig_file_actions(sig_act):
+ """Process signature file update actions."""
+ versions = get_local_versions()
+ sig_vers = versions.pronom_version
+ update_url = versions.update_site
+ if not update_url.endswith('/'):
+ update_url += '/'
+ if sig_act == 'check':
+ is_new, latest = _version_check(sig_vers, update_url)
+ if is_new:
+ print('Updated signatures v{} are available, current version is v{}'.format(latest, sig_vers))
+ else:
+ print('Your signature files are up to date, current version is v{}'.format(sig_vers))
+ elif sig_act == 'list':
+ resp = requests.get(update_url + 'format/')
+ tree = ET.fromstring(resp.content)
+ print('Available signature versions')
+ for child in tree.iter('signature'):
+ print(child.get('version'))
+ elif sig_act == 'update':
+ is_new, latest = _version_check(sig_vers, update_url)
+ if not is_new:
+ print('Your signature files are up to date, current version is v{}'.format(sig_vers))
+ sys.exit(0)
+ print('Updated signatures v{} are available, current version is v{}'.format(latest, sig_vers))
+ _output_details(latest, update_url, versions)
+ else:
+ ver = sig_act
+ if not ver.startswith('v'):
+ ver = 'v' + sig_act
+ resp = requests.get(update_url + 'format/' + ver + '/')
+ if resp.status_code != 200:
+ print('No signature files found for {}, REST status {}'.format(sig_act, resp.status_code))
+ sys.exit(1)
+ _output_details(re.search('\d+|$', ver).group(), update_url, versions) # noqa: W605
+
+
+def _output_details(version, update_url, versions):
+ print('Updating signatures')
+ _write_sigs(version, update_url, 'fido', 'formats-v{}.xml')
+ _write_sigs(version, update_url, 'droid', 'DROID_SignatureFile-{}.xml')
+ _write_sigs(version, update_url, 'fido', 'pronom-xml-{}.zip')
+ versions.pronom_version = '{}'.format(version)
+ versions.pronom_signature = 'pronom-xml-{}.zip'.format(version)
+ versions.write()
+
+
+def _version_check(sig_ver, update_url):
+ resp = requests.get(update_url + 'format/latest/')
+ if resp.status_code != 200:
+ print('Error getting latest version info {}'.format(resp.status_code))
+ sys.exit(1)
+ latest = re.search('\d+|$', resp.text).group() # noqa: W605
+ return int(latest) > int(sig_ver), latest
+
+
+def _write_sigs(latest, update_url, type, name_template):
+ sig_out = str(importlib_resources.files('fido').joinpath('conf', name_template.format(latest)))
+ if os.path.exists(sig_out):
+ return
+ resp = requests.get(update_url + 'format/latest/{}/'.format(type))
+ open(sig_out, 'wb').write(resp.content)
diff --git a/setup.py b/setup.py
index 161e1b7..f5fa790 100755
--- a/setup.py
+++ b/setup.py
@@ -29,6 +29,7 @@ def find_version(*file_paths):
'olefile >= 0.46, < 1',
'six >= 1.10.0, < 2',
'win-unicode-console >= 0.5; python_version == "2.7" and platform_system == "Windows"',
+ 'importlib-resources',
]
@@ -59,7 +60,7 @@ def find_version(*file_paths):
tests_require=tests_require,
extras_require=EXTRAS,
packages=['fido'],
- package_data={'fido': ['*.*', 'conf/*.*']},
+ package_data={'fido': ['*.*', 'conf/*.*', 'signatures/*.*', 'pronom/*.*']},
entry_points={'console_scripts': [
'fido = fido.fido:main',
'fido-prepare = fido.prepare:main',