From 513431fa5954e5f7bf687a8f334587929a9d1b8f Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:00:36 +0100 Subject: [PATCH 01/23] Update for Python 3.12 --- .github/workflows/main.yml | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ec7515..a866a4a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.10", "3.12"] runs-on: ubuntu-latest @@ -54,7 +54,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.12" - name: Install and upgrade dependencies run: | diff --git a/setup.py b/setup.py index 3fdc869..a40d164 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,9 @@ long_description = fh.read() setup(name="pipelinewise-singer-python", - version='2.0.1', + version='3.0.0', description="Singer.io utility library - PipelineWise compatible", - python_requires=">=3.7.0, <3.11", + python_requires="==3.12", long_description=long_description, long_description_content_type="text/markdown", author="TransferWise", From 0fd14c5d11e904f18201deadecc88f3a03bc094c Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:04:57 +0100 Subject: [PATCH 02/23] Update for Python 3.12 --- .github/workflows/main.yml | 6 +++--- .github/workflows/pythonpublish.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a866a4a..88e8b8d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,13 +14,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.12"] + python-version: [3.12"] runs-on: ubuntu-latest steps: - name: Checking out repo - uses: actions/checkout@v3 + uses: actions/checkout@v4.1.7 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -39,7 +39,7 @@ jobs: run: coverage run --parallel -m pytest - name: Upload coverage data - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.1.7 with: name: coverage-data path: ".coverage.*" diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index ffcea55..5688124 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -8,9 +8,9 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4.1.7 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v4.1.7 with: python-version: '3.x' - name: Install dependencies From ed3b9ef5ed416f30ce9bc3bdd24422e9a0cfb775 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:08:33 +0100 Subject: [PATCH 03/23] Update for Python 3.12 --- .github/workflows/pythonpublish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 5688124..b0e5028 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v4.1.7 - name: Set up Python - uses: actions/setup-python@v4.1.7 + uses: actions/setup-python@v5.1.0 with: python-version: '3.x' - name: Install dependencies From e4318121e4a6d50d9c44a6501ba399923c9471aa Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:09:37 +0100 Subject: [PATCH 04/23] Update for Python 3.12 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 88e8b8d..91c0785 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,7 +39,7 @@ jobs: run: coverage run --parallel -m pytest - name: Upload coverage data - uses: actions/upload-artifact@v4.1.7 + uses: actions/upload-artifact@v3 with: name: coverage-data path: ".coverage.*" From 02280115e353ec0594c229d4dd3fae6dfa4d9fd2 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:11:56 +0100 Subject: [PATCH 05/23] Update for Python 3.12 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91c0785..2645fd3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,7 +39,7 @@ jobs: run: coverage run --parallel -m pytest - name: Upload coverage data - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v7 with: name: coverage-data path: ".coverage.*" From eae9b2776516a21707785b72e9d7c71def002951 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:13:53 +0100 Subject: [PATCH 06/23] Update for Python 3.12 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2645fd3..f5a568c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.12"] + python-version: ["3.12"] runs-on: ubuntu-latest From cf5338282b5dd0931e66dd101aac3e314be309a8 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:17:17 +0100 Subject: [PATCH 07/23] Update for Python 3.12 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a40d164..35e67a6 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup(name="pipelinewise-singer-python", version='3.0.0', description="Singer.io utility library - PipelineWise compatible", - python_requires="==3.12", + python_requires=">=3.12, <3.13", long_description=long_description, long_description_content_type="text/markdown", author="TransferWise", From 5932ae1b9275daa14fa58c96e59f274c3c3ace14 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:31:19 +0100 Subject: [PATCH 08/23] Update for Python 3.12 --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 35e67a6..ca424d2 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,11 @@ 'Programming Language :: Python :: 3 :: Only' ], url="https://github.com/transferwise/pipelinewise-singer-python", + setup_requires=[ + 'wrapt>=1.14.0', + ], install_requires=[ + 'wrapt>=1.14.0', 'pytz', 'jsonschema==3.2.0', 'orjson==3.7.2', From ffc493c057be7da23f950e07248a07f3dc5b7b2b Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:35:18 +0100 Subject: [PATCH 09/23] Update for Python 3.12 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ca424d2..2213309 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ 'wrapt>=1.14.0', 'pytz', 'jsonschema==3.2.0', - 'orjson==3.7.2', + 'orjson==3.11.8', 'python-dateutil>=2.6.0', 'backoff==2.1.2', 'ciso8601', From 9efd7f9a7b9374aeb436b85b42ccc8409b388f9f Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:39:08 +0100 Subject: [PATCH 10/23] Update for Python 3.12 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2213309..bcbf069 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ ], extras_require={ 'dev': [ - 'pylint==2.11.1', - 'pytest==7.1.2', + 'pylint==4.0.5', + 'pytest==9.0.3', 'coverage[toml]~=6.3', 'ipython', 'ipdb', From 2da52d1ef598fa3f5b24e9147dd3845b4b82740c Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 16:50:17 +0100 Subject: [PATCH 11/23] Update for Python 3.12 --- pylintrc | 361 +++---------------------------------------------------- 1 file changed, 14 insertions(+), 347 deletions(-) diff --git a/pylintrc b/pylintrc index 35ad045..53e079e 100644 --- a/pylintrc +++ b/pylintrc @@ -1,71 +1,32 @@ -# Based on Apache 2.0 licensed code from https://github.com/ClusterHQ/flocker - [MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -# init-hook= - -# Add files or directories to the blacklist. They should be base names, not paths. ignore= - -# Pickle collected data for later comparisons. persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. load-plugins= - -# Use multiple processes to speed up Pylint. -# DO NOT CHANGE THIS VALUES >1 HIDE RESULTS!!!!! jobs=1 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist=ujson - -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no - +# Renamed from extension-pkg-whitelist +extension-pkg-allow-list=ujson [MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED confidence= -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. +# Fixed typos: 'noo-else-return' -> 'no-else-return' disable=wrong-import-order, broad-except, missing-module-docstring, - duplicate-code, # not useful until a major code refactoring - + duplicate-code, c-extension-no-member, missing-class-docstring, missing-function-docstring, - noo-else-return, + no-else-return, too-few-public-methods, too-many-arguments, too-many-branches, too-many-return-statements, - protected-access, - + protected-access +# Removed all removed/deprecated Python 2 checks (e.g., backtick, hex-method, etc.) +# Added missing commas where you had line breaks without them enable=import-error, import-self, reimported, @@ -78,7 +39,6 @@ enable=import-error, used-before-assignment, cell-var-from-loop, global-variable-undefined, - redefine-in-handler, unused-import, unused-wildcard-import, global-variable-not-assigned, @@ -87,7 +47,7 @@ enable=import-error, global-at-module-level, bad-open-mode, redundant-unittest-assert, - boolean-datetime + boolean-datetime, deprecated-method, anomalous-unicode-escape-in-string, anomalous-backslash-in-string, @@ -111,7 +71,7 @@ enable=import-error, assert-on-tuple, dangerous-default-value, duplicate-key, - useless-else-on-loop + useless-else-on-loop, expression-not-assigned, confusing-with-statement, unnecessary-lambda, @@ -123,10 +83,6 @@ enable=import-error, exec-used, using-constant-test, bad-super-call, - missing-super-argument, - slots-on-old-class, - super-on-old-class, - property-on-old-class, not-an-iterable, not-a-mapping, format-needs-mapping, @@ -142,36 +98,11 @@ enable=import-error, bad-format-string, missing-format-attribute, missing-format-argument-key, - unused-format-string-argument + unused-format-string-argument, unused-format-string-key, invalid-format-index, bad-indentation, - mixed-indentation, unnecessary-semicolon, - lowercase-l-suffix, - invalid-encoded-data, - unpacking-in-except, - import-star-module-level, - long-suffix, - old-octal-literal, - old-ne-operator, - backtick, - old-raise-syntax, - metaclass-assignment, - next-method-called, - dict-iter-method, - dict-view-method, - indexing-exception, - raising-string, - using-cmp-argument, - cmp-method, - coerce-method, - delslice-method, - getslice-method, - hex-method, - nonzero-method, - t-method, - setslice-method, logging-format-truncated, logging-too-few-args, logging-too-many-args, @@ -216,347 +147,83 @@ enable=import-error, raising-non-exception, misplaced-bare-raise, duplicate-except, - nonstandard-exception, binary-op-exception, bare-except, not-async-context-manager, yield-inside-async-function -# Needs investigation: -# abstract-method (might be indicating a bug? probably not though) -# protected-access (requires some refactoring) -# attribute-defined-outside-init (requires some refactoring) -# super-init-not-called (requires some cleanup) - -# Things we'd like to enable someday: -# redefined-builtin (requires a bunch of work to clean up our code first) -# redefined-outer-name (requires a bunch of work to clean up our code first) -# undefined-variable (re-enable when pylint fixes https://github.com/PyCQA/pylint/issues/760) -# no-name-in-module (giving us spurious warnings https://github.com/PyCQA/pylint/issues/73) -# unused-argument (need to clean up or code a lot, e.g. prefix unused_?) -# function-redefined (@overload causes lots of spurious warnings) -# too-many-function-args (@overload causes spurious warnings... I think) -# parameter-unpacking (needed for eventual Python 3 compat) -# print-statement (needed for eventual Python 3 compat) -# filter-builtin-not-iterating (Python 3) -# map-builtin-not-iterating (Python 3) -# range-builtin-not-iterating (Python 3) -# zip-builtin-not-iterating (Python 3) -# many others relevant to Python 3 -# unused-variable (a little work to cleanup, is all) - -# ... [REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. output-format=parseable - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - [LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format logging-modules=logging - [FORMAT] - -# Maximum number of characters on a single line. max-line-length=120 - -# Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - [TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. ignored-modules=orjson - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. ignored-classes= - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. generated-members= - [VARIABLES] - -# Tells whether we should check for unused import in __init__ files. init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. callbacks=cb_,_cb - [SIMILARITIES] - -# Minimum lines number of a similarity. min-similarity-lines=4 - -# Ignore comments when computing similarities. ignore-comments=yes - -# Ignore docstrings when computing similarities. ignore-docstrings=yes - -# Ignore imports when computing similarities. ignore-imports=no - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - [MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX - [BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input - -# Good variable names which should always be accepted, separated by a comma +# Removed deprecated bad-functions and all the hint options good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Regular expression matching correct function names function-rgx=[a-z_][a-z0-9_]{2,40}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,40}$ - -# Regular expression matching correct variable names variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names method-rgx=[a-z_][a-z0-9_]{2,80}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,80}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. no-docstring-rgx=^_ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. docstring-min-length=-1 - -[ELIF] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - [DESIGN] - -# Maximum number of arguments for function / method max-args=7 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore ignored-argument-names=_.* - -# Maximum number of locals for function / method body max-locals=15 - -# Maximum number of return / yield for function / method body max-returns=6 - -# Maximum number of branch for function / method body max-branches=12 - -# Maximum number of statements in function / method body max-statements=50 - -# Maximum number of parents for a class (see R0901). max-parents=7 - -# Maximum number of attributes for a class (see R0902). max-attributes=7 - -# Minimum number of public methods for a class (see R0903). min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). max-public-methods=20 - -# Maximum number of boolean expressions in a if statement max-bool-expr=5 - [CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs - -# List of member names, which should be excluded from the protected access -# warning. exclude-protected=_asdict,_fields,_replace,_source,_make - [EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception +# Updated to fully qualified name +overgeneral-exceptions=builtins.Exception \ No newline at end of file From 4fb83df0ad80366d8b66d9d18b15f981ab632009 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 17:02:58 +0100 Subject: [PATCH 12/23] Update for Python 3.12 --- singer/messages.py | 8 ++++---- singer/metadata.py | 2 +- singer/transform.py | 2 +- singer/utils.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/singer/messages.py b/singer/messages.py index d0294ca..93f838f 100644 --- a/singer/messages.py +++ b/singer/messages.py @@ -12,7 +12,7 @@ class Message(): '''Base class for messages.''' def asdict(self): # pylint: disable=no-self-use - raise Exception('Not implemented') + raise RuntimeError('Not implemented') def __eq__(self, other): return isinstance(other, Message) and self.asdict() == other.asdict() @@ -97,7 +97,7 @@ def __init__(self, stream, schema, key_properties, bookmark_properties=None): if isinstance(bookmark_properties, (str, bytes)): bookmark_properties = [bookmark_properties] if bookmark_properties and not isinstance(bookmark_properties, list): - raise Exception('bookmark_properties must be a string or list of strings') + raise TypeError('bookmark_properties must be a string or list of strings') self.bookmark_properties = bookmark_properties @@ -226,7 +226,7 @@ def asdict(self): def _required_key(msg, k): if k not in msg: - raise Exception(f"Message is missing required key '{k}': {msg}") + raise ValueError(f"Message is missing required key '{k}': {msg}") return msg[k] @@ -333,7 +333,7 @@ def write_schema(stream_name, schema, key_properties, bookmark_properties=None, if isinstance(key_properties, (str, bytes)): key_properties = [key_properties] if not isinstance(key_properties, list): - raise Exception('key_properties must be a string or list of strings') + raise TypeError('key_properties must be a string or list of strings') write_message( SchemaMessage( diff --git a/singer/metadata.py b/singer/metadata.py index 41153ea..ce4f02d 100644 --- a/singer/metadata.py +++ b/singer/metadata.py @@ -12,7 +12,7 @@ def delete(compiled_metadata, breadcrumb, k): def write(compiled_metadata, breadcrumb, k, val): if val is None: - raise Exception() + raise ValueError() if breadcrumb in compiled_metadata: compiled_metadata.get(breadcrumb).update({k: val}) else: diff --git a/singer/transform.py b/singer/transform.py index f117570..7bb6c1b 100644 --- a/singer/transform.py +++ b/singer/transform.py @@ -229,7 +229,7 @@ def _transform_datetime(self, value): return None # Short circuit in the case of null or empty string if self.integer_datetime_fmt not in VALID_DATETIME_FORMATS: - raise Exception('Invalid integer datetime parsing option') + raise ValueError('Invalid integer datetime parsing option') if self.integer_datetime_fmt == NO_INTEGER_DATETIME_PARSING: return string_to_datetime(value) diff --git a/singer/utils.py b/singer/utils.py index 7579280..16fa1e6 100644 --- a/singer/utils.py +++ b/singer/utils.py @@ -66,7 +66,7 @@ def strptime_to_utc(dtimestr): def strftime(dtime, format_str=DATETIME_FMT): if dtime.utcoffset() != datetime.timedelta(0): - raise Exception('datetime must be pegged at UTC tzoneinfo') + raise ValueError('datetime must be pegged at UTC tzoneinfo') dt_str = None try: @@ -187,7 +187,7 @@ def parse_args(required_config_keys): def check_config(config, required_keys): missing_keys = [key for key in required_keys if key not in config] if missing_keys: - raise Exception(f'Config is missing required keys: {missing_keys}') + raise RuntimeError(f'Config is missing required keys: {missing_keys}') def backoff(exceptions, giveup): From 3e525987a42c738301d0fc1bfcce7c750bba094b Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 17:08:08 +0100 Subject: [PATCH 13/23] Update for Python 3.12 --- singer/catalog.py | 3 ++- singer/messages.py | 6 +++--- singer/schema.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/singer/catalog.py b/singer/catalog.py index 77424a9..52efc51 100644 --- a/singer/catalog.py +++ b/singer/catalog.py @@ -25,7 +25,8 @@ class CatalogEntry(): def __init__(self, tap_stream_id=None, stream=None, key_properties=None, schema=None, replication_key=None, is_view=None, database=None, table=None, row_count=None, - stream_alias=None, metadata=None, replication_method=None): + stream_alias=None, + metadata=None, replication_method=None): # pylint: disable=too-many-positional-arguments self.tap_stream_id = tap_stream_id self.stream = stream diff --git a/singer/messages.py b/singer/messages.py index 93f838f..e6c8247 100644 --- a/singer/messages.py +++ b/singer/messages.py @@ -11,7 +11,7 @@ class Message(): '''Base class for messages.''' - def asdict(self): # pylint: disable=no-self-use + def asdict(self): raise RuntimeError('Not implemented') def __eq__(self, other): @@ -196,7 +196,7 @@ class BatchMessage(Message): def __init__( self, stream, filepath, file_format=None, compression=None, batch_size=None, time_extracted=None - ): + ): # pylint: disable=too-many-positional-arguments self.stream = stream self.filepath = filepath self.format = file_format or 'jsonl' @@ -363,7 +363,7 @@ def write_version(stream_name, version): def write_batch( stream_name, filepath, file_format=None, compression=None, batch_size=None, time_extracted=None -): +): # pylint: disable=too-many-positional-arguments """Write a batch message. stream = 'users' diff --git a/singer/schema.py b/singer/schema.py index 108f50f..9378cd2 100644 --- a/singer/schema.py +++ b/singer/schema.py @@ -36,7 +36,7 @@ def __init__(self, type=None, format=None, properties=None, items=None, selected=None, inclusion=None, description=None, minimum=None, maximum=None, exclusiveMinimum=None, exclusiveMaximum=None, multipleOf=None, maxLength=None, minLength=None, additionalProperties=None, - anyOf=None, patternProperties=None): + anyOf=None, patternProperties=None): # pylint: disable=too-many-positional-arguments self.type = type self.properties = properties From fa89dac8fa1e54c313b129708acb8e1c3cfc9eb3 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 17:11:07 +0100 Subject: [PATCH 14/23] Update for Python 3.12 --- singer/catalog.py | 4 +++- singer/schema.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/singer/catalog.py b/singer/catalog.py index 52efc51..0702da3 100644 --- a/singer/catalog.py +++ b/singer/catalog.py @@ -22,11 +22,13 @@ def write_catalog(catalog): # pylint: disable=too-many-instance-attributes class CatalogEntry(): + # pylint: disable=too-many-positional-arguments + def __init__(self, tap_stream_id=None, stream=None, key_properties=None, schema=None, replication_key=None, is_view=None, database=None, table=None, row_count=None, stream_alias=None, - metadata=None, replication_method=None): # pylint: disable=too-many-positional-arguments + metadata=None, replication_method=None): self.tap_stream_id = tap_stream_id self.stream = stream diff --git a/singer/schema.py b/singer/schema.py index 9378cd2..c38ccc2 100644 --- a/singer/schema.py +++ b/singer/schema.py @@ -31,12 +31,13 @@ class Schema(): # pylint: disable=too-many-instance-attributes ''' + # pylint: disable=too-many-positional-arguments # pylint: disable=too-many-locals def __init__(self, type=None, format=None, properties=None, items=None, selected=None, inclusion=None, description=None, minimum=None, maximum=None, exclusiveMinimum=None, exclusiveMaximum=None, multipleOf=None, maxLength=None, minLength=None, additionalProperties=None, - anyOf=None, patternProperties=None): # pylint: disable=too-many-positional-arguments + anyOf=None, patternProperties=None): self.type = type self.properties = properties From a598608bd6d202c1c4d7b1bbc6c749a8cebec4aa Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 17:15:33 +0100 Subject: [PATCH 15/23] Update for Python 3.12 --- tests/test_catalog.py | 6 +++--- tests/test_schema.py | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index cd6dc50..8a72e1a 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -25,7 +25,7 @@ def test_one_selected_stream(self): CatalogEntry(tap_stream_id='c',schema=Schema(),metadata=[])]) state = {} selected_streams = catalog.get_selected_streams(state) - self.assertEquals([e for e in selected_streams],[selected_entry]) + self.assertEqual([e for e in selected_streams],[selected_entry]) def test_resumes_currently_syncing_stream(self): selected_entry_a = CatalogEntry(tap_stream_id='a', @@ -44,7 +44,7 @@ def test_resumes_currently_syncing_stream(self): selected_entry_c]) state = {'currently_syncing': 'c'} selected_streams = catalog.get_selected_streams(state) - self.assertEquals([e for e in selected_streams][0],selected_entry_c) + self.assertEqual([e for e in selected_streams][0],selected_entry_c) class TestToDictAndFromDict(unittest.TestCase): @@ -141,4 +141,4 @@ def test(self): CatalogEntry(tap_stream_id='b'), CatalogEntry(tap_stream_id='c')]) entry = catalog.get_stream('b') - self.assertEquals('b', entry.tap_stream_id) + self.assertEqual('b', entry.tap_stream_id) diff --git a/tests/test_schema.py b/tests/test_schema.py index bf9edc6..abf7abd 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -44,41 +44,41 @@ class TestSchema(unittest.TestCase): additionalProperties=True) def test_to_string(self): - self.assertEquals('{"maxLength":32,"type":"string"}', str(self.string_obj)) + self.assertEqual('{"maxLength":32,"type":"string"}', str(self.string_obj)) def test_string_to_dict(self): - self.assertEquals(self.string_dict, self.string_obj.to_dict()) + self.assertEqual(self.string_dict, self.string_obj.to_dict()) def test_integer_to_dict(self): - self.assertEquals(self.integer_dict, self.integer_obj.to_dict()) + self.assertEqual(self.integer_dict, self.integer_obj.to_dict()) def test_array_to_dict(self): - self.assertEquals(self.array_dict, self.array_obj.to_dict()) + self.assertEqual(self.array_dict, self.array_obj.to_dict()) def test_object_to_dict(self): - self.assertEquals(self.object_dict, self.object_obj.to_dict()) + self.assertEqual(self.object_dict, self.object_obj.to_dict()) def test_string_from_dict(self): - self.assertEquals(self.string_obj, Schema.from_dict(self.string_dict)) + self.assertEqual(self.string_obj, Schema.from_dict(self.string_dict)) def test_integer_from_dict(self): - self.assertEquals(self.integer_obj, Schema.from_dict(self.integer_dict)) + self.assertEqual(self.integer_obj, Schema.from_dict(self.integer_dict)) def test_array_from_dict(self): - self.assertEquals(self.array_obj, Schema.from_dict(self.array_dict)) + self.assertEqual(self.array_obj, Schema.from_dict(self.array_dict)) def test_object_from_dict(self): - self.assertEquals(self.object_obj, Schema.from_dict(self.object_dict)) + self.assertEqual(self.object_obj, Schema.from_dict(self.object_dict)) def test_repr_atomic(self): - self.assertEquals(self.string_obj, eval(repr(self.string_obj))) + self.assertEqual(self.string_obj, eval(repr(self.string_obj))) def test_repr_recursive(self): - self.assertEquals(self.object_obj, eval(repr(self.object_obj))) + self.assertEqual(self.object_obj, eval(repr(self.object_obj))) def test_object_from_dict_with_defaults(self): schema = Schema.from_dict(self.object_dict, inclusion='automatic') - self.assertEquals('whatever', schema.inclusion, + self.assertEqual('whatever', schema.inclusion, msg='The schema value should override the default') - self.assertEquals('automatic', schema.properties['a_string'].inclusion) - self.assertEquals('automatic', schema.properties['an_array'].items.inclusion) + self.assertEqual('automatic', schema.properties['a_string'].inclusion) + self.assertEqual('automatic', schema.properties['an_array'].items.inclusion) From 88653e0f314d907214817a0b2592a53797945184 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 17:18:49 +0100 Subject: [PATCH 16/23] Update for Python 3.12 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f5a568c..2d93e9e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,7 +62,7 @@ jobs: python3 -m pip install -U .[dev] - name: Download coverage data - uses: actions/download-artifact@v3.0.0 + uses: actions/download-artifact@v7 with: name: coverage-data From 83d2de9df28d2366e91e70adb5f121e8ecae0d43 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Thu, 16 Apr 2026 17:27:48 +0100 Subject: [PATCH 17/23] fix action --- .github/workflows/main.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d93e9e..09a33de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,16 +43,18 @@ jobs: with: name: coverage-data path: ".coverage.*" + include-hidden-files: true + if-no-files-found: error coverage: runs-on: ubuntu-latest needs: build steps: - name: Check out the repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.12" @@ -62,13 +64,14 @@ jobs: python3 -m pip install -U .[dev] - name: Download coverage data - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: coverage-data - name: Combine coverage data run: | - coverage combine + coverage combine .coverage.* + coverage report - name: Generate XML coverage report run: | From 5303c3ee49492e465c018a5bc16458736e994cb1 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 17 Apr 2026 09:09:50 +0100 Subject: [PATCH 18/23] add some unit tests --- tests/test_utils.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index fb19f92..8cff8f6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -18,6 +18,40 @@ def test_round_trip(self): self.assertEqual(dtime, fdtime) + def test_parse_naive_datetime(self): + """Test that a naive datetime string is assigned UTC.""" + dt_string = "2026-04-17 09:00:00" + result = u.strptime_with_tz(dt_string) + self.assertIsNotNone(result.tzinfo) + self.assertEqual(result.tzinfo, pytz.UTC) + self.assertEqual(result.year, 2026) + self.assertEqual(result.hour, 9) + + def test_parse_aware_datetime(self): + """Test that an existing timezone is preserved (not overwritten by UTC).""" + # Using an offset of +02:00 + dt_string = "2026-04-17 09:00:00+02:00" + result = u.strptime_with_tz(dt_string) + + self.assertIsNotNone(result.tzinfo) + # The offset should be 120 minutes (2 hours) + self.assertEqual(result.utcoffset().total_seconds(), 7200) + self.assertNotEqual(result.tzinfo, pytz.UTC) + + def test_parse_iso_format(self): + """Test that standard ISO formats work correctly.""" + dt_string = "2026-04-17T12:30:45Z" + result = u.strptime_with_tz(dt_string) + + self.assertEqual(result.hour, 12) + # 'Z' is UTC + self.assertEqual(result.tzinfo.utcoffset(result).total_seconds(), 0) + + def test_invalid_string_raises_error(self): + """Test that completely invalid strings still raise parser errors.""" + with self.assertRaises(Exception): + u.strptime_with_tz("not-a-date") + class TestHandleException(unittest.TestCase): def setUp(self): self.logger = logging.getLogger(__name__) From ef2835e80992bbbec4327ed5b9329cdb3269fdb8 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 17 Apr 2026 09:15:16 +0100 Subject: [PATCH 19/23] add some unit tests --- tests/test_utils.py | 69 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 8cff8f6..28cfc81 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,7 @@ import pytz import logging import singer.utils as u - +from unittest.mock import patch, call class TestFormat(unittest.TestCase): def test_small_years(self): @@ -52,6 +52,73 @@ def test_invalid_string_raises_error(self): with self.assertRaises(Exception): u.strptime_with_tz("not-a-date") + @patch('time.time') + @patch('time.sleep') + def test_ratelimit_sleeps_when_limit_exceeded(self, mock_sleep, mock_time): + """Verify that sleep is called correctly after the limit is reached.""" + # Setup: Limit of 2 calls every 10 seconds + limit = 2 + every = 10 + + @u.ratelimit(limit, every) + def mock_func(): + return True + + # First call at T=0 + # Second call at T=1 + # Third call at T=2 (This should trigger a sleep) + mock_time.side_effect = [0.0, 1.0, 2.0, 3.0] + + # Call 1: Success, no sleep + mock_func() + # Call 2: Success, no sleep + mock_func() + + # Call 3: This exceeds the limit of 2. + # It compares current time (2.0) with the oldest time (0.0). + # elapsed = 2.0 - 0.0 = 2.0. + # sleep_time = 10 - 2.0 = 8.0. + mock_func() + + # Check if sleep was called with 8.0 seconds + mock_sleep.assert_called_once_with(8.0) + + @patch('time.time') + @patch('time.sleep') + def test_ratelimit_no_sleep_if_enough_time_passed(self, mock_sleep, mock_time): + """Verify no sleep occurs if the time elapsed exceeds the 'every' window.""" + limit = 2 + every = 10 + + @u.ratelimit(limit, every) + def mock_func(): + return True + + # Call 1 at T=0 + # Call 2 at T=1 + # Call 3 at T=15 (15s passed since Call 1, which is > 'every' window of 10s) + mock_time.side_effect = [0.0, 1.0, 15.0, 16.0] + + mock_func() + mock_func() + mock_func() + + # Sleep should NOT have been called + mock_sleep.assert_not_called() + + def test_decorator_preserves_metadata(self): + """Ensure functools.wraps is working to keep function names.""" + + @u.ratelimit(5, 60) + def original_function(): + """Docstring""" + pass + + self.assertEqual(original_function.__name__, "original_function") + self.assertEqual(original_function.__doc__, "Docstring") + + + class TestHandleException(unittest.TestCase): def setUp(self): self.logger = logging.getLogger(__name__) From 000c9509d002368229028f9d123ccd2179d66e47 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 17 Apr 2026 10:02:57 +0100 Subject: [PATCH 20/23] fix actions --- .github/workflows/pythonpublish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index b0e5028..ae9b250 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.1.0 with: - python-version: '3.x' + python-version: '3.12' - name: Install dependencies run: | python -m pip install --upgrade pip From d8271151bb05c2542321296d1699edee26f5cc1e Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 17 Apr 2026 12:16:56 +0100 Subject: [PATCH 21/23] move ownership to AP --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..c803ccf --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default approvers +* @transferwise/analytics-platform \ No newline at end of file From a7827955a4cde8339ea9f5eff0e7e16a86275d27 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 17 Apr 2026 12:56:42 +0100 Subject: [PATCH 22/23] undo move ownership to AP --- .github/CODEOWNERS | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index c803ccf..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,2 +0,0 @@ -# Default approvers -* @transferwise/analytics-platform \ No newline at end of file From a754614efddc53261390ec62181531547d6c8712 Mon Sep 17 00:00:00 2001 From: Amir Mofakhar Date: Fri, 17 Apr 2026 14:37:47 +0100 Subject: [PATCH 23/23] move ownership --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..c803ccf --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default approvers +* @transferwise/analytics-platform \ No newline at end of file