jrid:0020
- GL Transmission Format: GLTF runtime 3D asset library (Generic)
+ GL Transmission Format: GLTF runtime 3D asset library schema (Generic)
False
False
False
True
- {'KEY': '$schema', 'STARTSWITH': 'http://json-schema.org/'}
-{'KEY': '$schema', 'ENDSWITH': '/schema#'}
+ {'KEY': '$schema', 'STARTSWITH': 'https://json-schema.org/'}
+{'KEY': '$schema', 'ENDSWITH': '/schema'}
{'KEY': 'title', 'EXISTS': None}
{'KEY': 'type', 'IS': 'object'}
{'KEY': 'description', 'IS': 'The root object for a glTF asset.'}
@@ -1510,7 +1510,7 @@ Contents
jrid:0017: Open Resume Document jrid:0018: jacker song jrid:0019: JSON Patch jrid:0020: GL Transmission Format: GLTF runtime 3D asset library (Generic) jrid:0020: GL Transmission Format: GLTF runtime 3D asset library schema (Generic) jrid:0021: Tweet Object jrid:0022: sandboxels save file jrid:0023: dublin core metadata (archivematica) json2json will convert JSON compatible objects from one encoding
to UTF-8.
src.utils.jsonid2pronom
+jsonid2pronom provides a helper script to enable export of generic
+JSONID compatible markers to a PRONOM compatible signature file.
@@ -67,6 +72,7 @@
diff --git a/docs/utils/json2json.html b/docs/utils/json2json.html
index d68fa96..62dce2d 100644
--- a/docs/utils/json2json.html
+++ b/docs/utils/json2json.html
@@ -78,7 +78,7 @@
async def identify_json(paths: list[str]):
"""Identify objects."""
for idx, path in enumerate(paths):
- valid, data, _, _ = await identify_plaintext_bytestream(path)
+ valid, data = await identify_plaintext_bytestream(path)
if not valid:
continue
print(json.dumps(data, indent=2))src.utils.jsonid2pronom API documentation
+
+
+
+Module src.utils.jsonid2pronom
+
+
+jsonid2pronom provides a helper script to enable export of generic
+JSONID compatible markers to a PRONOM compatible signature file.
+
+
+
+
+
+
+
+async def load_patterns (path: str) ‑> list
+
+
+
+Expand source code
+
+async def load_patterns(path: str) -> list:
+ """Load patterns from a file for conversion to a signature file."""
+ patterns = []
+ with open(path, "r", encoding="utf-8") as patterns_file:
+ patterns = json.loads(patterns_file.read())
+ return patterns
+Load patterns from a file for conversion to a signature file.
+
+def main () ‑> None
+
+
+
+Expand source code
+
+def main() -> None:
+ """Primary entry point for this script."""
+ parser = argparse.ArgumentParser(
+ prog="jsonid2pronom",
+ description="convert JSONID compatible markers to PRONOM",
+ epilog="for more information visit https://github.com/ffdev-info/jsonid",
+ )
+ parser.add_argument(
+ "--debug",
+ help="use debug loggng",
+ required=False,
+ action="store_true",
+ )
+ parser.add_argument(
+ "--path",
+ "-p",
+ help="file path to process",
+ required=False,
+ )
+ args = parser.parse_args()
+ logging.getLogger(__name__).setLevel(logging.DEBUG if args.debug else logging.INFO)
+ logger.debug("debug logging is configured")
+ if not args.path:
+ parser.print_help(sys.stderr)
+ sys.exit()
+ asyncio.run(
+ output_signature(
+ path=args.path,
+ )
+ )
+Primary entry point for this script.
+
+async def output_signature (path: str)
+
+
+
+Expand source code
+
+async def output_signature(path: str):
+ """Output JSONID compatible signatures to PRONOM."""
+
+ formats = []
+
+ encodings = ("UTF-8", "UTF-16", "UTF-16BE", "UTF-32LE")
+ priorities = []
+
+ increment_id = 0
+
+ markers = await load_patterns(path)
+
+ if not markers:
+ logger.error("no patterns provided via path arg")
+ sys.exit(1)
+
+ for encoding in encodings:
+ increment_id += 1
+ json_puid = "jsonid2pronom/1"
+ name_ = f"JSONID2PRONOM Conversion ({encoding})"
+ try:
+ mime = "application/json"
+ except IndexError:
+ mime = ""
+ try:
+ sequences = pronom.process_markers(
+ copy.deepcopy(markers),
+ increment_id,
+ encoding=encoding,
+ )
+ except pronom.UnprocessableEntity as err:
+ logger.error(
+ "%s %s: cannot handle: %s",
+ json_puid,
+ name_,
+ err,
+ )
+ for err_marker in markers:
+ logger.debug("--- START ---")
+ logger.debug("marker: %s", err_marker)
+ logger.debug("--- END ---")
+ continue
+ fmt = pronom.Format(
+ id=increment_id,
+ name=name_,
+ version="",
+ puid=json_puid,
+ mime=mime,
+ classification="structured text",
+ external_signatures=[
+ pronom.ExternalSignature(
+ id=increment_id,
+ signature="json",
+ type=pronom.EXT,
+ )
+ ],
+ internal_signatures=sequences,
+ priorities=list(set(priorities)),
+ )
+ priorities.append(f"{increment_id}")
+ formats.append(fmt)
+
+ pronom.process_formats_to_stdout(formats)
+Output JSONID compatible signatures to PRONOM.
+
+
+
+
+
+
+
+
+
diff --git a/json2json.py b/json2json.py
new file mode 100644
index 0000000..1bcbe2b
--- /dev/null
+++ b/json2json.py
@@ -0,0 +1,12 @@
+"""JSONID entry-points."""
+
+from src.utils import json2json
+
+
+def main():
+ """Primary entry point for this script."""
+ json2json.main()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/json2pronom.py b/json2pronom.py
new file mode 100644
index 0000000..0f2c3f7
--- /dev/null
+++ b/json2pronom.py
@@ -0,0 +1,12 @@
+"""JSONID entry-points."""
+
+from src.utils import jsonid2pronom
+
+
+def main():
+ """Primary entry point for this script."""
+ jsonid2pronom.main()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/jsonid.py b/jsonid.py
index 5c087ae..597256b 100644
--- a/jsonid.py
+++ b/jsonid.py
@@ -1,4 +1,4 @@
-"""Donovan entry-points."""
+"""JSONID entry-points."""
from src.jsonid import jsonid
diff --git a/justfile b/justfile
index bfb4a32..9be2689 100644
--- a/justfile
+++ b/justfile
@@ -83,6 +83,10 @@ check-debug:
@hexdump file:
hexdump -v -e '"\\\x" 1/1 "%02x"' {{file}}
+# Hexdump bytes only.
+@hexdump-plain file:
+ hexdump -ve '1/1 "%.2x"' {{file}}
+
# code coverage
coverage:
python -m tox -e coverage
@@ -102,3 +106,25 @@ lookup-ref-ex:
# lookup ref
@lookup-ref ref:
python jsonid.py lookup {{ref}}
+
+# export as PRONOM
+pronom:
+ python jsonid.py --pronom --debug
+
+# move pronom to .droid folder
+pronom-to-droid:
+ ~/.droid6/clean.sh
+ rm -f ~/.droid6/signature_files/jsonid_pronom.xml
+ cp jsonid_pronom.xml ~/.droid6/signature_files/
+
+dir := `pwd`
+
+# load pronom to siegfried via roy
+roy:
+ @echo {{dir}}
+ roy build --noreports -extend {{dir}}/jsonid_pronom.xml
+
+# droid-cli
+droid-cli path: pronom-to-droid
+ java -jar ~/dp/droid/droid-binary-6.8.0-bin/droid-command-line-6.8.0.jar -s 1
+ java -jar ~/dp/droid/droid-binary-6.8.0-bin/droid-command-line-6.8.0.jar -ri {{path}}
diff --git a/pronom_example/patterns_example.json b/pronom_example/patterns_example.json
new file mode 100644
index 0000000..8631d38
--- /dev/null
+++ b/pronom_example/patterns_example.json
@@ -0,0 +1,4 @@
+[
+ {"KEY": "key1", "EXISTS": null},
+ {"KEY": "key2", "EXISTS": null}
+]
diff --git a/pyproject.toml b/pyproject.toml
index b7fe1b5..991bcdf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,6 +40,7 @@ dependencies = {file = ["requirements/requirements.txt"]}
jsonid = "jsonid.jsonid:main"
momoa = "jsonid.jsonid:main"
json2json = "utils.json2json:main"
+json2pronom = "utils.jsonid2pronom:main"
[build-system]
requires = ["setuptools>=80.9.0", "wheel>=0.45.1", "setuptools_scm[toml]>=9.2.2"]
diff --git a/src/jsonid/export.py b/src/jsonid/export.py
index c062aa8..7164a0c 100644
--- a/src/jsonid/export.py
+++ b/src/jsonid/export.py
@@ -1,25 +1,35 @@
"""Functions to support export."""
+import copy
import datetime
import json
import logging
from datetime import timezone
+from typing import Final
try:
+ import pronom
import registry_data
import version
except ModuleNotFoundError:
try:
- from src.jsonid import registry_data, version
+ from src.jsonid import pronom, registry_data, version
except ModuleNotFoundError:
- from jsonid import registry_data, version
+ from jsonid import pronom, registry_data, version
logger = logging.getLogger(__name__)
+PRONOM_FILENAME: Final[str] = "jsonid_pronom.xml"
+
+
+class PRONOMException(Exception):
+ """Exception class if we can't create a PRONOM signature as expected."""
+
+
def exportJSON() -> None: # pylint: disable=C0103
"""Export to JSON."""
- logger.debug("exporting registry ad JSON")
+ logger.debug("exporting registry as JSON")
data = registry_data.registry()
json_obj = []
id_ = {
@@ -35,3 +45,102 @@ def exportJSON() -> None: # pylint: disable=C0103
for datum in data:
json_obj.append(datum.json())
print(json.dumps(json_obj, indent=2))
+
+
+def export_pronom() -> None:
+ """Export a PRONOM compatible set of signatures.
+
+ Export is done in two phases. A set of proposed "Baseline" JSON
+ signatures to catch many JSON instances.
+
+ Second the JSONID registry is exported.
+
+ Every export has a priority over the other so that there should
+ be no multiple identification results.
+ """
+
+ # pylint: disable=R0914; too-many local variables.
+
+ logger.debug("exporting registry as PRONOM")
+
+ reg_data = registry_data.registry()
+ formats = []
+
+ encodings = ("UTF-8", "UTF-16", "UTF-16BE", "UTF-32LE")
+ priorities = []
+
+ increment_id = 0
+
+ for encoding in encodings:
+ all_baseline = pronom.create_baseline_json_sequences(encoding)
+ for baseline in all_baseline:
+ increment_id += 1
+ fmt = pronom.Format(
+ id=increment_id,
+ name=f"JSON (Baseline - fmt/817) ({encoding})",
+ version="",
+ puid="jsonid:0000",
+ mime="application/json",
+ classification="structured text",
+ external_signatures=[
+ pronom.ExternalSignature(
+ id=increment_id,
+ signature="json",
+ type=pronom.EXT,
+ )
+ ],
+ internal_signatures=[baseline],
+ priorities=priorities,
+ )
+ priorities.append(f"{increment_id}")
+ formats.append(fmt)
+
+ for encoding in encodings:
+ for entry in reg_data:
+ increment_id += 1
+ json_puid = f"{entry.json()['identifier']};{encoding}"
+ name_ = f"{entry.json()['name'][0]['@en']} ({encoding})"
+ markers = entry.json()["markers"]
+ try:
+ mime = entry.json()["mime"][0]
+ except IndexError:
+ mime = ""
+ try:
+ sequences = pronom.process_markers(
+ copy.deepcopy(markers),
+ increment_id,
+ encoding=encoding,
+ )
+ except pronom.UnprocessableEntity as err:
+ logger.error(
+ "%s %s: cannot handle: %s",
+ json_puid,
+ name_,
+ err,
+ )
+ for marker in markers:
+ logger.debug("--- START ---")
+ logger.debug("marker: %s", marker)
+ logger.debug("--- END ---")
+ continue
+ fmt = pronom.Format(
+ id=increment_id,
+ name=name_,
+ version="",
+ puid=json_puid,
+ mime=mime,
+ classification="structured text",
+ external_signatures=[
+ pronom.ExternalSignature(
+ id=increment_id,
+ signature="json",
+ type=pronom.EXT,
+ )
+ ],
+ internal_signatures=sequences,
+ priorities=copy.deepcopy(list(set(priorities))),
+ )
+ priorities.append(f"{increment_id}")
+ formats.append(fmt)
+
+ pronom.process_formats_and_save(formats, PRONOM_FILENAME)
diff --git a/src/jsonid/export_helpers.py b/src/jsonid/export_helpers.py
new file mode 100644
index 0000000..0dfec2f
--- /dev/null
+++ b/src/jsonid/export_helpers.py
@@ -0,0 +1,30 @@
+"""Helpers for the export functions."""
+
+import datetime
+from datetime import timezone
+from typing import Final
+from xml.dom.minidom import parseString
+
+UTC_TIME_FORMAT: Final[str] = "%Y-%m-%dT%H:%M:%SZ"
+
+
+def get_utc_timestamp_now():
+ """Get a formatted UTC timestamp for 'now' that can be used when
+ a timestamp is needed.
+ """
+ return datetime.datetime.now(timezone.utc).strftime(UTC_TIME_FORMAT)
+
+
+def new_prettify(c):
+ """Remove excess newlines from DOM output.
+
+ via: https://stackoverflow.com/a/14493981
+ """
+ reparsed = parseString(c)
+ return "\n".join(
+ [
+ line
+ for line in reparsed.toprettyxml(indent=" " * 2).split("\n")
+ if line.strip()
+ ]
+ )
diff --git a/src/jsonid/file_processing.py b/src/jsonid/file_processing.py
index 1d083a4..1da44ee 100644
--- a/src/jsonid/file_processing.py
+++ b/src/jsonid/file_processing.py
@@ -338,6 +338,27 @@ async def identify_plaintext_bytestream(
If analysis is `True` we try to return more low-level file
information to help folks make appraisal decisions.
+
+ Encodings in Python are split into the following, where UTF-32 on
+ its own is a little confusing. If WE are writing the encoding then
+ I believe it leaves off the byte-order-marker and we want to
+ select UTF-32LE to make sure it is written.
+
+ If we are decoding, then I don't think it matters. I think we
+ try to decode and if it works it works.
+
+ Encodings:
+
+ "UTF-8",
+ "UTF-16",
+ "UTF-16LE",
+ "UTF-16BE",
+ "UTF-32",
+ "UTF-32LE",
+ "UTF-32BE",
+ "SHIFT-JIS",
+ "BIG5",
+
"""
# pylint: disable=R0911
diff --git a/src/jsonid/helpers.py b/src/jsonid/helpers.py
index ca28bc1..3776fc1 100644
--- a/src/jsonid/helpers.py
+++ b/src/jsonid/helpers.py
@@ -2,7 +2,7 @@
import logging
import time
-from typing import Union
+from typing import Final, Union
try:
import htm_template
@@ -75,6 +75,15 @@ def format_marker(marker_text: str, marker: dict) -> str:
return f"{marker_text}{marker_formatted}\n"
+TYPE_BOOL: Final[str] = "bool"
+TYPE_FLOAT: Final[str] = "float"
+TYPE_INTEGER: Final[str] = "integer"
+TYPE_LIST: Final[str] = "list"
+TYPE_NONE: Final[str] = "NoneType"
+TYPE_MAP: Final[str] = "map"
+TYPE_STRING: Final[str] = "string"
+
+
def substitute_type_text(replace_me: Union[str, type]):
"""Output a text substitution for a type that will otherwise not
pretty-print.
@@ -83,19 +92,19 @@ def substitute_type_text(replace_me: Union[str, type]):
# pylint: disable=R0911
if replace_me.__name__ == "dict":
- return "map"
+ return TYPE_MAP
if replace_me.__name__ == "int":
- return "integer"
+ return TYPE_INTEGER
if replace_me.__name__ == "list":
- return "list"
+ return TYPE_LIST
if replace_me.__name__ == "str":
- return "string"
+ return TYPE_STRING
if replace_me.__name__ == "float":
- return "float"
+ return TYPE_FLOAT
if replace_me.__name__ == "bool":
- return "bool"
+ return TYPE_BOOL
if replace_me.__name__ == "NoneType":
- return "NoneType"
+ return TYPE_NONE
if not isinstance(replace_me, type):
pass
return replace_me
diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py
index 0dfa6b8..7475e0d 100644
--- a/src/jsonid/jsonid.py
+++ b/src/jsonid/jsonid.py
@@ -98,7 +98,7 @@ def main() -> None:
# pylint: disable=R0912,R0915
parser = argparse.ArgumentParser(
- prog="json-id",
+ prog="jsonid",
description="proof-of-concept identifier for JSON objects on disk based on identifying valid objects and their key-values",
epilog="for more information visit https://github.com/ffdev-info/json-id",
)
@@ -156,10 +156,12 @@ def main() -> None:
help="path to a custom registry to lead into memory replacing the default",
required=False,
)
+ # NB. consider output to stdout once the feature is more stable.
parser.add_argument(
"--pronom",
- help="return a PRONOM-centric view of the results",
+ help=f"return a PRONOM-centric view of the results to `{export.PRONOM_FILENAME}` (BETA)",
required=False,
+ action="store_true",
)
parser.add_argument(
"--export",
@@ -220,7 +222,8 @@ def main() -> None:
if args.registry:
raise NotImplementedError("custom registry is not yet available")
if args.pronom:
- raise NotImplementedError("pronom view is not yet implemented")
+ export.export_pronom()
+ sys.exit()
if args.language:
raise NotImplementedError("multiple languages are not yet implemented")
if args.export:
diff --git a/src/jsonid/pronom.py b/src/jsonid/pronom.py
new file mode 100644
index 0000000..5a279df
--- /dev/null
+++ b/src/jsonid/pronom.py
@@ -0,0 +1,686 @@
+"""PRONOM export routines.
+
+XML tooling: https://xmllint.com/
+"""
+
+import binascii
+import codecs
+import logging
+import xml.dom.minidom
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Any, Final
+
+try:
+ import export_helpers
+ import helpers
+ import registry_matchers
+except ModuleNotFoundError:
+ try:
+ from src.jsonid import export_helpers, helpers, registry_matchers
+ except ModuleNotFoundError:
+ from jsonid import export_helpers, helpers, registry_matchers
+
+
+logger = logging.getLogger(__name__)
+
+
+DISK_SECTOR_SIZE: Final[int] = 4095
+
+# Common PRONOM characters.
+COLON: Final[str] = "3A"
+CURLY_OPEN: Final[str] = "7B"
+CURLY_CLOSE: Final[str] = "7D"
+SQUARE_OPEN: Final[str] = "5B"
+SQUARE_CLOSE: Final[str] = "5D"
+DOUBLE_QUOTE: Final[str] = "22"
+
+# Constant values.
+NUMBER_REGEX: Final[str] = "[30:39]"
+TRUE_VALUE: Final[str] = "74727565"
+FALSE_VALUE: Final[str] = "66616C7365"
+NULL_VALUE: Final[str] = "6E756C6C"
+
+# Our whitespace values could potentially be optimized per encoding,
+# e.g. to be more or less per encoding. 16 is a good default to enable
+# some UTF32-identification.
+WHITESPACE_REGEX: Final[str] = "{0-16}"
+
+# External signature types.
+EXT: Final[str] = "file extension"
+
+# Replacement markers for PRONOM pre-processing.
+MARKER_INDEX_START = "INDEX_START"
+MARKER_INDEX_END = "INDEX END"
+
+
+class UnprocessableEntity(Exception):
+ """Provide a way to give complete feedback to the caller to allow
+ it to exit."""
+
+
+@dataclass
+class ExternalSignature:
+ id: str
+ signature: str
+ type: str
+
+
+@dataclass
+class ByteSequence:
+ id: str
+ pos: str
+ min_off: str
+ max_off: str
+ endian: str
+ value: str
+
+
+@dataclass
+class InternalSignature:
+ id: str
+ name: str
+ byte_sequences: list[ByteSequence]
+
+
+@dataclass
+class Priority:
+ type: str
+ id: str
+
+
+@dataclass
+class Identifier:
+ type: str
+ value: str
+
+
+@dataclass
+class Format: # pylint: disable=R0902
+ id: str
+ name: str
+ version: str
+ puid: str
+ mime: str
+ classification: str
+ external_signatures: list[ExternalSignature]
+ internal_signatures: list[InternalSignature]
+ priorities: list[int]
+
+
+@lru_cache()
+def _get_bom() -> list:
+ """Generate a list of byte-order markers that allow us to replace
+ markers introduced through various encoding operations.
+ """
+ replaces = [
+ codecs.BOM,
+ codecs.BOM_BE,
+ codecs.BOM_LE,
+ codecs.BOM_UTF8,
+ codecs.BOM_UTF16,
+ codecs.BOM_UTF16_BE,
+ codecs.BOM_UTF16_LE,
+ codecs.BOM_UTF32,
+ codecs.BOM_UTF32_BE,
+ codecs.BOM_UTF32_LE,
+ ]
+ res = []
+ for bom in replaces:
+ hex_bom = ""
+ for marker in bom:
+ char = hex(marker)
+ hex_bom = f"{hex_bom}{char.replace('0x', '')}".upper()
+ res.append(hex_bom)
+ return res
+
+
+def create_many_to_one_byte_sequence(internal_signatures: list[InternalSignature]):
+ """Create a many to one byte sequence, i.e. a format with multiple
+ Internal Signatures.
+ """
+ internal_signature = ""
+ for internal in internal_signatures:
+ id_ = internal.id
+ bs = create_one_to_many_byte_sequence(internal.byte_sequences)
+ internal_signature = f"""
+{internal_signature}
+ {bs}
+
+ """
+ return internal_signature.strip()
+
+
+def calculate_variable_off_bof(item: ByteSequence):
+ """Given variable offsets, calculate the correct syntax."""
+ seq = item.value
+ if (
+ item.min_off != ""
+ and int(item.min_off) > 0
+ and item.max_off != ""
+ and int(item.max_off) > 0
+ ):
+ seq = f"{{{item.min_off}-{int(item.min_off)+int(item.max_off)}}}{seq}"
+ elif item.max_off != "" and int(item.max_off) > 0:
+ seq = f"{{0-{item.max_off}}}{seq}"
+ elif item.min_off != "" and int(item.min_off) > 0:
+ seq = f"{{{item.min_off}}}{seq}"
+ return seq
+
+
+def calculate_variable_off_eof(item: ByteSequence):
+ """Given variable offsets, calculate the correct syntax."""
+ seq = item.value
+ if (
+ item.min_off != ""
+ and int(item.min_off) > 0
+ and item.max_off != ""
+ and int(item.max_off) > 0
+ ):
+ seq = f"{seq}{{{item.min_off}-{int(item.min_off)+int(item.max_off)}}}"
+ elif item.max_off != "" and int(item.max_off) > 0:
+ seq = f"{seq}{{0-{item.max_off}}}"
+ elif item.min_off != "" and int(item.min_off) > 0:
+ seq = f"{seq}{{{item.min_off}}}"
+ return seq
+
+
+def create_one_to_many_byte_sequence(byte_sequences: list[ByteSequence]):
+ """Create a byte sequence object."""
+ byte_sequence = ""
+ for item in byte_sequences:
+ seq = item.value
+ if item.pos.startswith("EOF"):
+ seq = calculate_variable_off_eof(item)
+ elif item.pos.startswith("BOF"):
+ seq = calculate_variable_off_bof(item)
+ byte_sequence = f"""
+{byte_sequence.strip()}
+
+ """
+ return byte_sequence.strip()
+
+
+def create_file_format_collection(fmt: list[Format]):
+ """Create the FileFormatCollection object.
+
+ E.g.
+ ```
+
+ 1
+ ext
+
+
+
+ 880
+ 881
+ ai
+ 86
+ 331
+ 332
+ 771
+ 773
+
+ ```
+
+ """
+ internal_sigs = [
+ f"{sig.id} "
+ for sig in fmt.internal_signatures
+ ]
+ external_sigs = [
+ f"{sig.signature} "
+ for sig in fmt.external_signatures
+ if sig.type.lower() == EXT
+ ]
+ priorities = [
+ f"{priority} "
+ for priority in fmt.priorities
+ ]
+ ff = f"""
+
+ {"".join(internal_sigs).strip()}
+ {"".join(external_sigs).strip()}
+ {"".join(priorities).strip()}
+
+ """
+ return ff.strip()
+
+
+def _process_formats(formats: list[Format]):
+ """Process formats into a PRONOM XML file."""
+ isc = []
+ ffc = []
+ for fmt in formats:
+ ffc.append(create_file_format_collection(fmt))
+ if fmt.internal_signatures:
+ isc.append(create_many_to_one_byte_sequence(fmt.internal_signatures))
+ droid_template = f"""
+
+
+
+ {"".join(isc).strip()}
+
+
+ {"".join(ffc).strip()}
+
+
+ """
+ dom = None
+ signature_file = droid_template.strip().replace("\n", "")
+ try:
+ dom = xml.dom.minidom.parseString(signature_file)
+ except xml.parsers.expat.ExpatError as err:
+ logger.error("cannot process xml: %s", err)
+ return ""
+ pretty_xml = dom.toprettyxml(indent=" ", encoding="utf-8")
+ prettier_xml = export_helpers.new_prettify(pretty_xml)
+ return prettier_xml
+
+
+def process_formats_and_save(formats: list[Format], filename: str):
+ """Process the collected formats and output a signature file.
+
+ NB. Given our dataclasses here, we have the opportunity to rework
+ this data into many new structures. We output XML because DROID
+ expects XML.
+ """
+ prettier_xml = _process_formats(formats)
+ logger.info("outputting to: %s", filename)
+ with open(filename, "w", encoding="utf=8") as output_file:
+ output_file.write(prettier_xml)
+
+
+def process_formats_to_stdout(formats: list[Format]):
+ """Process the collected formats and output a signature file.
+
+ NB. Given our dataclasses here, we have the opportunity to rework
+ this data into many new structures. We output XML because DROID
+ expects XML.
+ """
+ prettier_xml = _process_formats(formats)
+ logger.info("outputting to: stdout")
+ print(prettier_xml)
+
+
+def encode_roundtrip(hexed_val: str, encoding: str) -> str:
+ """We want to get a plain-text byte-sequence into a new
+ encoding. It takes a few hops and skips.
+ """
+ val = hexed_val.strip()
+ try:
+ re_encoded = binascii.unhexlify(hexed_val).decode("utf-8").encode(encoding)
+ except (binascii.Error, UnicodeDecodeError) as err:
+ logger.error("cannot convert: %s len: %s ('%s')", hexed_val, len(val), err)
+ return val
+ hex_val = binascii.hexlify(re_encoded).decode().upper()
+ for bom in _get_bom():
+ if not hex_val.startswith(bom):
+ continue
+ return hex_val.replace(bom, "")
+ return hex_val
+
+
+def _type_to_str(type_: type, encoding: str) -> str:
+ """Given a data type marker we need to convert the type into a
+ byte sequence that will match the type.
+
+ E.g. BOOLEAN types evaluate to true or false encoded in ASCII.
+ E.g. STRING types need to begin and end with double-quotes but the
+ string itself is just a wildcard. The wildcard will match any
+ value between the double quotes.
+ """
+
+ curly_open_encoded = encode_roundtrip(CURLY_OPEN, encoding)
+ curly_close_encoded: Final[str] = encode_roundtrip(CURLY_CLOSE, encoding)
+ square_open_encoded: Final[str] = encode_roundtrip(SQUARE_OPEN, encoding)
+ square_close_encoded: Final[str] = encode_roundtrip(SQUARE_CLOSE, encoding)
+ double_quote_encoded: Final[str] = encode_roundtrip(DOUBLE_QUOTE, encoding)
+
+ try:
+ type_ = helpers.substitute_type_text(type_)
+ except AttributeError:
+ logger.debug("type_ already converted: %s", type_)
+
+ if type_ in (helpers.TYPE_INTEGER, type_ == helpers.TYPE_FLOAT):
+ # an integer field will begin 0-9 but it is unclear how to
+ # represent larger numbers? and whether we need to?
+ return NUMBER_REGEX
+ if type_ == helpers.TYPE_BOOL:
+ # true | false
+ return f"({encode_roundtrip(TRUE_VALUE, encoding)}|{encode_roundtrip(FALSE_VALUE, encoding)})"
+ if type_ == helpers.TYPE_STRING:
+ # string begins with a double quote and ends in a double quote.
+ return f"'{double_quote_encoded}*{double_quote_encoded}"
+ if type_ == helpers.TYPE_MAP:
+ # { == 7B; } == 7D
+ return f"{curly_open_encoded}*{curly_close_encoded}"
+ if type_ == helpers.TYPE_LIST:
+ # [ == 5B; ] == 5D
+ return f"{square_open_encoded}*{square_close_encoded}"
+ if type_ == helpers.TYPE_NONE:
+ # null
+ return f"{encode_roundtrip(NULL_VALUE, encoding)}".encode(encoding)
+ # This should only trigger for incorrect values at this point..
+ raise UnprocessableEntity(f"type_to_str: {type_}")
+
+
+def _complex_is_type(marker: Any) -> str:
+ """Complex IS might be another data structure, e.g. a dict, or
+ something else that we can't convert easily. It is simply a WIP
+ for now.
+ """
+ raise UnprocessableEntity(f"complex IS type: '{marker}' (WIP)")
+
+
+def _str_to_hex_str(string: str) -> str:
+ """Convert string to hexadecimal bytes.
+
+ We convert to bytes here first without encoding and then convert
+ the bytes to an encoding second. It should be possible to combine
+ those two procedures, but this has worked well during the
+ prototyping phase.
+ """
+ hex_bytes = []
+ for byte_ in string.encode():
+ hex_bytes.append(hex(byte_).replace("0x", ""))
+ hex_str = "".join(hex_bytes).upper()
+ return hex_str
+
+
+def quote_and_encode(value, encoding) -> str:
+ """Quote and encode a given value."""
+
+ double_quote_encoded: Final[str] = encode_roundtrip(DOUBLE_QUOTE, encoding)
+ # return f"{double_quote_encoded}{value}{double_quote_encoded}"
+ return f"{double_quote_encoded}{encode_roundtrip(value, encoding)}{double_quote_encoded}"
+
+
+def convert_marker_to_signature_sequence(marker: dict, encoding: str) -> str:
+ """Convert a JSONID marker into a signature sequence."""
+
+ # pylint: disable=R0914; too-many local variables.
+ # pylint: disable=R0911; too-many return statements.
+ # pylint: disable=R0915; too-many statements.
+
+ logger.debug("marker: %s", marker)
+
+ colon_encoded: Final[str] = encode_roundtrip(COLON, encoding)
+ double_quote_encoded: Final[str] = encode_roundtrip(DOUBLE_QUOTE, encoding)
+ curly_open_encoded: Final[str] = encode_roundtrip(CURLY_OPEN, encoding)
+ curly_close_encoded: Final[str] = encode_roundtrip(CURLY_CLOSE, encoding)
+ colon_encoded: Final[str] = encode_roundtrip(COLON, encoding)
+ square_open_encoded: Final[str] = encode_roundtrip(SQUARE_OPEN, encoding)
+ square_close_encoded: Final[str] = encode_roundtrip(SQUARE_CLOSE, encoding)
+ double_quote_encoded: Final[str] = encode_roundtrip(DOUBLE_QUOTE, encoding)
+
+ instruction = ""
+ if registry_matchers.MARKER_GOTO in marker.keys():
+ # GOTO KEY and match KEY.
+ goto_key = _str_to_hex_str(marker["GOTO"])
+ key_at_goto = _str_to_hex_str(marker["KEY"])
+ goto_encoded = quote_and_encode(goto_key, encoding)
+ key_encoded = quote_and_encode(key_at_goto, encoding)
+ instruction = f"{goto_encoded}{WHITESPACE_REGEX}{colon_encoded}*{WHITESPACE_REGEX}{key_encoded}{WHITESPACE_REGEX}{colon_encoded}"
+ marker.pop("GOTO")
+ marker.pop("KEY")
+ return instruction.upper()
+ if registry_matchers.MARKER_INDEX in marker.keys():
+ key = _str_to_hex_str(marker["KEY"])
+ instruction = f"{WHITESPACE_REGEX}{square_open_encoded}*{curly_open_encoded}*{double_quote_encoded}{encode_roundtrip(key, encoding)}{double_quote_encoded}{WHITESPACE_REGEX}{colon_encoded}*{curly_close_encoded}*{square_close_encoded}"
+ marker.pop("INDEX")
+ marker.pop("KEY")
+ return instruction.upper()
+ if "KEY" in marker.keys():
+ key = _str_to_hex_str(marker["KEY"])
+ instruction = quote_and_encode(key, encoding)
+ marker.pop("KEY")
+ if registry_matchers.MARKER_KEY_EXISTS in marker.keys():
+ instruction = f"{instruction}{WHITESPACE_REGEX}{colon_encoded}".upper()
+ return instruction
+ if registry_matchers.MARKER_IS_TYPE in marker.keys():
+ is_type = _type_to_str(marker["ISTYPE"], encoding=encoding)
+ type_val = (
+ f"{instruction}{WHITESPACE_REGEX}{colon_encoded}{WHITESPACE_REGEX}{is_type}"
+ )
+ return type_val.upper()
+ if registry_matchers.MARKER_IS in marker.keys():
+ marker_is = marker["IS"]
+ if not isinstance(marker_is, str):
+ _complex_is_type(marker_is)
+ equals = _str_to_hex_str(marker_is)
+ is_val = f"{instruction}{WHITESPACE_REGEX}{encode_roundtrip(equals, encoding)}"
+ return is_val.upper()
+ if registry_matchers.MARKER_STARTSWITH in marker.keys():
+ starts_with = _str_to_hex_str(marker["STARTSWITH"])
+ starts_with_val = f"{instruction}{WHITESPACE_REGEX}{colon_encoded}{WHITESPACE_REGEX}{double_quote_encoded}{encode_roundtrip(starts_with, encoding)}"
+ return starts_with_val.upper()
+ if registry_matchers.MARKER_ENDSWITH in marker.keys():
+ ends_with = _str_to_hex_str(marker["ENDSWITH"])
+ ends_with_val = f"{instruction}{WHITESPACE_REGEX}{colon_encoded}{WHITESPACE_REGEX}*{encode_roundtrip(ends_with, encoding)}{double_quote_encoded}"
+ return ends_with_val.upper()
+ if registry_matchers.MARKER_CONTAINS in marker.keys():
+ contains = _str_to_hex_str(marker["CONTAINS"])
+ contains_val = f"{instruction}{WHITESPACE_REGEX}{colon_encoded}{WHITESPACE_REGEX}{double_quote_encoded}*{encode_roundtrip(contains, encoding)}*{double_quote_encoded}"
+ return contains_val.upper()
+ if registry_matchers.MARKER_REGEX in marker.keys():
+ raise UnprocessableEntity("REGEX not yet implemented")
+ if registry_matchers.MARKER_KEY_NO_EXIST in marker.keys():
+ raise UnprocessableEntity("KEY NO EXIST not yet implemented")
+ # We should never arrive here. In the future clean this up so we
+ # only return when we have information.
+ return ""
+
+
+def preprocess_goto_markers(markers: dict) -> list:
+ """Preprocess markers to remove data that is otherwise duplicated
+ when converted to a PRONOM signature, e.g. GOTO."""
+
+ out = []
+ for marker in markers:
+ if registry_matchers.MARKER_GOTO not in marker:
+ out.append(marker)
+ continue
+ key = marker.pop("GOTO")
+ new_marker = {"KEY": key, "EXISTS": None}
+ if new_marker not in out:
+ out.append(new_marker)
+ out.append(marker)
+ return out
+
+
+def process_markers(
+ markers: list, sig_id: int, encoding: str = ""
+) -> tuple[list | bool]:
+ """Given a set of markers for a document type, process them into
+ a set of byte sequences and finally an internal signature sequence
+ that can be output as a PRONOM signature.
+
+ returns a tuple describing the processed value and a flag to
+ highlight the result is potentially lossless, e.g. in the case
+ of matching types, e.g. strings.
+
+ dict_keys(['CONTAINS'])
+ dict_keys(['ENDSWITH'])
+ dict_keys(['IS']
+ dict_keys(['ISTYPE'])
+ dict_keys(['STARTSWITH'])
+
+ key(0-n):(0-n)value
+
+ Need to return something like:
+
+