🔖 bump version 1.0.8 -> 1.1.0

Instead of parsing the entire log file at once, use a generator and return JSON lines.

.gitignore

@@ -160,6 +160,6 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 # End of https://www.toptal.com/developers/gitignore/api/python

.idea/civiproxy_logs2json.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (civiproxy_logs2json) (2)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

.idea/misc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (civiproxy_logs2json)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (civiproxy_logs2json) (2)" project-jdk-type="Python SDK" />
 </project>

MANIFEST.in

@@ -0,0 +1 @@
+include src/civiproxy_logs2json/*.toml

README.md

@@ -9,9 +9,9 @@ python3 -m pip install civiproxy_logs2json --user
 
 ## Example Usage
 
-Pass logfile as positional argument:
+Pass logfile as option:
 ```bash
-cpl2j /var/www/proxy_logs/proxy.log
+cpl2j -f /var/www/proxy_logs/proxy.log
 ```
 
 Pipe logfile into program:
@@ -24,9 +24,14 @@ Set JSON indentation to two spaces:
 cat proxy.log | cpl2j -s 2 
 ```
 
+Output JSON lines:
+```bash
+cpl2j -l -f /var/www/proxy_logs/proxy.log
+```
+
 ### Tip
 Use [VisiData](https://github.com/saulpw/visidata) to explore the data in a very comfortable way:
 /var/www/proxy_logs/proxy.log
 ```bash
-cpl2j /var/www/proxy_logs/proxy.log | vd -f json
+cpl2j -l -f /var/www/proxy_logs/proxy.log | vd -f jsonl
 ```

civiproxy_logs2json/__init__.py

@@ -1 +0,0 @@
-from .__main__ import main, translate_logfile

civiproxy_logs2json/__main__.py

@@ -1,63 +0,0 @@
-import argparse
-import json
-import re
-import sys
-from typing import TextIO
-
-REQUEST_LINE_RE = r"^REQUEST FROM (?P<source>(\d{1,3}\.){3}\d{1,3}) ON (?P<date>\d\d\d\d-\d\d-\d\d) (?P<time>\d\d:\d\d:\d\d) -- Array$"
-VALUES_LINE_RE = r"^\s{4}\[(?P<key>.*)] => (?P<value>.*)$"
-CLOSING_LINE_RE = r"^\)$"
-
-# Setup argparse
-argparser = argparse.ArgumentParser(
-    description='Translate a CiviProxy logfile into JSON format. ')
-argparser.add_argument('-f',
-                       '--logfile',
-                       help='CiviProxy logfile',
-                       type=argparse.FileType('r', encoding='UTF-8'),
-                       default=(None if sys.stdin.isatty() else sys.stdin))
-argparser.add_argument('-i',
-                       '--indent',
-                       help='number of spaces to indent JSON output',
-                       type=int,
-                       default=4)
-
-
-def main():
-    args = argparser.parse_args()
-
-    # Print info if no logfile is specified or passed via stdin
-    if not args.logfile:
-        print('Please specify a path to a CiviProxy logfile')
-        sys.exit()
-
-    # Parse logfile and print it to console
-    print(json.dumps(translate_logfile(args.logfile), indent=args.indent))
-
-
-def translate_logfile(logfile: TextIO) -> list:
-    json_ = []
-    with logfile as file:
-        array = {}
-        for line in file:
-            request_line = re.search(REQUEST_LINE_RE, line)
-            values_line = re.search(VALUES_LINE_RE, line)
-            close_line = re.search(CLOSING_LINE_RE, line)
-            if request_line:
-                array["date"] = request_line.group("date")
-                array["time"] = request_line.group("time")
-                array["source"] = request_line.group("source")
-            elif values_line:
-                if values_line.group("key") == "json":
-                    array["values"] = json.loads(values_line.group("value"))
-                else:
-                    array[values_line.group("key")] = values_line.group("value")
-            elif close_line:
-                if array:
-                    json_.append(array)
-                    array = {}
-    return json_
-
-
-if __name__ == '__main__':
-    main()

pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "civiproxy_logs2json"
+version = "1.1.0"
+description = "Translate a CiviProxy logfile into JSON format."
+readme = "README.md"
+authors = [{ name = "Marc Michalsky", email = "michalsky@forumZFD.de" }]
+license = { file = "LICENSE" }
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+]
+keywords = ["civicrm", "json", "logs"]
+dependencies = ['tomli; python_version < "3.11"',]
+requires-python = ">=3.5"
+
+[project.optional-dependencies]
+dev = ["pytest", "bumpver"]
+
+[project.urls]
+Homepage = "https://github.com/MarcMichalsky/civiproxy_logs2json"
+
+[project.scripts]
+cpl2j = "civiproxy_logs2json.__main__:main"
+
+[tool.bumpver]
+current_version = "1.1.0"
+version_pattern = "MAJOR.MINOR.PATCH"
+commit_message = "🔖 bump version {old_version} -> {new_version}"
+commit = true
+tag = true
+push = false
+
+[tool.bumpver.file_patterns]
+"pyproject.toml" = [
+    '"{version}"',
+]
+"src/civiproxy_logs2json/__init__.py" = [
+    "{version}"
+]

setup.py

@@ -1,23 +1,3 @@
-from setuptools import setup, find_packages
+from setuptools import setup
 
-with open("README.md", "r") as fh:
+setup()
-    description = fh.read()
-
-setup(
-    name="civiproxy_logs2json",
-    version="1.0.3",
-    author="Marc Michalsky",
-    author_email="michalsky@forumZFD.de",
-    packages=find_packages("civiproxy_logs2json", exclude=["test"]),
-    description="Translate a CiviProxy logfile into JSON format.",
-    long_description=description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/MarcMichalsky/civiproxy_logs2json",
-    license='MIT',
-    python_requires='>=3.5',
-    entry_points={
-        'console_scripts': [
-            'cpl2j = civiproxy_logs2json.__main__:main',
-        ],
-    },
-)

src/civiproxy_logs2json/__init__.py

@@ -0,0 +1,12 @@
+from importlib import resources
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+__version__ = "1.1.0"
+
+_cfg = tomllib.loads(resources.read_text("civiproxy_logs2json", "config.toml"))
+REQUEST_LINE_RE = _cfg["regex"]["request_line"]
+VALUES_LINE_RE = _cfg["regex"]["values_line"]
+CLOSING_LINE_RE = _cfg["regex"]["closing_line"]

src/civiproxy_logs2json/__main__.py

@@ -0,0 +1,74 @@
+import argparse
+import json
+import re
+import sys
+from typing import TextIO
+
+from . import REQUEST_LINE_RE, VALUES_LINE_RE, CLOSING_LINE_RE
+
+
+# Setup argparse
+argparser = argparse.ArgumentParser(
+    description="Translate a CiviProxy logfile into JSON Lines format.")
+argparser.add_argument("-f",
+                       "--logfile",
+                       help="CiviProxy logfile",
+                       type=argparse.FileType("r", encoding="UTF-8"),
+                       default=(None if sys.stdin.isatty() else sys.stdin))
+argparser.add_argument("-i",
+                       "--indent",
+                       help="number of spaces to indent JSON output",
+                       type=int,
+                       default=None)
+argparser.add_argument("-l",
+                       "--lines",
+                       help="output JSON Lines instead of JSON",
+                       action="store_true")
+
+
+def main():
+    args = argparser.parse_args()
+
+    # Print info if no logfile is specified or passed via stdin
+    if not args.logfile:
+        print("Please specify a path to a CiviProxy logfile")
+        sys.exit()
+
+    # Parse logfile and print it to console
+    for line in translate_logfile(args.logfile,
+                                  indent=args.indent,
+                                  json_lines=args.lines):
+        print(line)
+
+
+def translate_logfile(logfile: TextIO, indent: int, json_lines: bool) -> str:
+    _json = []
+    array = {}
+    with logfile as file:
+        for line in file:
+            request_line = re.search(REQUEST_LINE_RE, line)
+            values_line = re.search(VALUES_LINE_RE, line)
+            close_line = re.search(CLOSING_LINE_RE, line)
+            if request_line:
+                array["date"] = request_line.group("date")
+                array["time"] = request_line.group("time")
+                array["source"] = request_line.group("source")
+            elif values_line:
+                if values_line.group("key") == "json":
+                    array["values"] = json.loads(values_line.group("value"))
+                else:
+                    array[values_line.group("key")] = values_line.group("value")
+            elif close_line:
+                if array:
+                    if json_lines:
+                        yield json.dumps(array, indent=indent)
+                        array = {}
+                    else:
+                        _json.append(array)
+                        array = {}
+        if not json_lines:
+            yield json.dumps(_json, indent=indent)
+
+
+if __name__ == "__main__":
+    main()

src/civiproxy_logs2json/config.toml

@@ -0,0 +1,4 @@
+[regex]
+request_line = "^REQUEST FROM (?P<source>(\\d{1,3}\\.){3}\\d{1,3}) ON (?P<date>\\d\\d\\d\\d-\\d\\d-\\d\\d) (?P<time>\\d\\d:\\d\\d:\\d\\d) -- Array$"
+values_line = "^\\s{4}\\[(?P<key>.*)] => (?P<value>.*)$"
+closing_line = "^\\)$"

tests/test_translate_logfile.py

@@ -1,6 +1,6 @@
 import os
 import json
-from civiproxy_logs2json import translate_logfile
+from src.civiproxy_logs2json.__main__ import translate_logfile
 
 
 def test_translate_logfile():