From a3dd4a6dbf4a1318f104ce1991e638220b7f248d Mon Sep 17 00:00:00 2001 From: Brendan Haines Date: Sat, 14 Sep 2024 12:45:24 -0600 Subject: [PATCH] working gotification and retries --- Pipfile | 1 + Pipfile.lock | 70 +++++++++++++++++++++++++++++++++++++++++++++---- goat_monitor.py | 50 ++++++++++++++++++++++++++++++++--- 3 files changed, 112 insertions(+), 9 deletions(-) diff --git a/Pipfile b/Pipfile index 5a20030..cf58b47 100644 --- a/Pipfile +++ b/Pipfile @@ -7,6 +7,7 @@ name = "pypi" gotify = "*" click = "*" toml = "*" +numpy = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 8accebb..4bf6c6b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0f54883c8ada6db11ca84430fcbd7e2c6faf74e1d4b2cad48c78b2d29aa3faf8" + "sha256": "13a000de8cd5f6c3d270d059f8df15653008a32b9daa31494cc26bd4c80dd8d0" }, "pipfile-spec": 6, "requires": { @@ -76,11 +76,71 @@ }, "idna": { "hashes": [ - "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac", - "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603" + "sha256:69297d5da0cc9281c77efffb4e730254dd45943f45bbfb461de5991713989b1e", + "sha256:e5c5dafde284f26e9e0f28f6ea2d6400abd5ca099864a67f576f3981c6476124" ], "markers": "python_version >= '3.6'", - "version": "==3.8" + "version": "==3.9" + }, + "numpy": { + "hashes": [ + "sha256:046356b19d7ad1890c751b99acad5e82dc4a02232013bd9a9a712fddf8eb60f5", + "sha256:0b8cc2715a84b7c3b161f9ebbd942740aaed913584cae9cdc7f8ad5ad41943d0", + "sha256:0d07841fd284718feffe7dd17a63a2e6c78679b2d386d3e82f44f0108c905550", + "sha256:13cc11c00000848702322af4de0147ced365c81d66053a67c2e962a485b3717c", + "sha256:13ce49a34c44b6de5241f0b38b07e44c1b2dcacd9e36c30f9c2fcb1bb5135db7", + "sha256:24c2ad697bd8593887b019817ddd9974a7f429c14a5469d7fad413f28340a6d2", + "sha256:251105b7c42abe40e3a689881e1793370cc9724ad50d64b30b358bbb3a97553b", + "sha256:2ca4b53e1e0b279142113b8c5eb7d7a877e967c306edc34f3b58e9be12fda8df", + "sha256:3269c9eb8745e8d975980b3a7411a98976824e1fdef11f0aacf76147f662b15f", + "sha256:397bc5ce62d3fb73f304bec332171535c187e0643e176a6e9421a6e3eacef06d", + "sha256:3fc5eabfc720db95d68e6646e88f8b399bfedd235994016351b1d9e062c4b270", + "sha256:50a95ca3560a6058d6ea91d4629a83a897ee27c00630aed9d933dff191f170cd", + "sha256:52ac2e48f5ad847cd43c4755520a2317f3380213493b9d8a4c5e37f3b87df504", + "sha256:53e27293b3a2b661c03f79aa51c3987492bd4641ef933e366e0f9f6c9bf257ec", + "sha256:57eb525e7c2a8fdee02d731f647146ff54ea8c973364f3b850069ffb42799647", + "sha256:5889dd24f03ca5a5b1e8a90a33b5a0846d8977565e4ae003a63d22ecddf6782f", + "sha256:59ca673ad11d4b84ceb385290ed0ebe60266e356641428c845b39cd9df6713ab", + "sha256:6435c48250c12f001920f0751fe50c0348f5f240852cfddc5e2f97e007544cbe", + "sha256:6e5a9cb2be39350ae6c8f79410744e80154df658d5bea06e06e0ac5bb75480d5", + "sha256:7be6a07520b88214ea85d8ac8b7d6d8a1839b0b5cb87412ac9f49fa934eb15d5", + "sha256:7c803b7934a7f59563db459292e6aa078bb38b7ab1446ca38dd138646a38203e", + "sha256:7dd86dfaf7c900c0bbdcb8b16e2f6ddf1eb1fe39c6c8cca6e94844ed3152a8fd", + "sha256:8661c94e3aad18e1ea17a11f60f843a4933ccaf1a25a7c6a9182af70610b2313", + "sha256:8ae0fd135e0b157365ac7cc31fff27f07a5572bdfc38f9c2d43b2aff416cc8b0", + "sha256:910b47a6d0635ec1bd53b88f86120a52bf56dcc27b51f18c7b4a2e2224c29f0f", + "sha256:913cc1d311060b1d409e609947fa1b9753701dac96e6581b58afc36b7ee35af6", + "sha256:920b0911bb2e4414c50e55bd658baeb78281a47feeb064ab40c2b66ecba85553", + "sha256:950802d17a33c07cba7fd7c3dcfa7d64705509206be1606f196d179e539111ed", + "sha256:981707f6b31b59c0c24bcda52e5605f9701cb46da4b86c2e8023656ad3e833cb", + "sha256:98ce7fb5b8063cfdd86596b9c762bf2b5e35a2cdd7e967494ab78a1fa7f8b86e", + "sha256:99f4a9ee60eed1385a86e82288971a51e71df052ed0b2900ed30bc840c0f2e39", + "sha256:9a8e06c7a980869ea67bbf551283bbed2856915f0a792dc32dd0f9dd2fb56728", + "sha256:ae8ce252404cdd4de56dcfce8b11eac3c594a9c16c231d081fb705cf23bd4d9e", + "sha256:afd9c680df4de71cd58582b51e88a61feed4abcc7530bcd3d48483f20fc76f2a", + "sha256:b49742cdb85f1f81e4dc1b39dcf328244f4d8d1ded95dea725b316bd2cf18c95", + "sha256:b5613cfeb1adfe791e8e681128f5f49f22f3fcaa942255a6124d58ca59d9528f", + "sha256:bab7c09454460a487e631ffc0c42057e3d8f2a9ddccd1e60c7bb8ed774992480", + "sha256:c8a0e34993b510fc19b9a2ce7f31cb8e94ecf6e924a40c0c9dd4f62d0aac47d9", + "sha256:caf5d284ddea7462c32b8d4a6b8af030b6c9fd5332afb70e7414d7fdded4bfd0", + "sha256:cea427d1350f3fd0d2818ce7350095c1a2ee33e30961d2f0fef48576ddbbe90f", + "sha256:d0cf7d55b1051387807405b3898efafa862997b4cba8aa5dbe657be794afeafd", + "sha256:d10c39947a2d351d6d466b4ae83dad4c37cd6c3cdd6d5d0fa797da56f710a6ae", + "sha256:d2b9cd92c8f8e7b313b80e93cedc12c0112088541dcedd9197b5dee3738c1201", + "sha256:d4c57b68c8ef5e1ebf47238e99bf27657511ec3f071c465f6b1bccbef12d4136", + "sha256:d51fc141ddbe3f919e91a096ec739f49d686df8af254b2053ba21a910ae518bf", + "sha256:e097507396c0be4e547ff15b13dc3866f45f3680f789c1a1301b07dadd3fbc78", + "sha256:e30356d530528a42eeba51420ae8bf6c6c09559051887196599d96ee5f536468", + "sha256:e8d5f8a8e3bc87334f025194c6193e408903d21ebaeb10952264943a985066ca", + "sha256:e8dfa9e94fc127c40979c3eacbae1e61fda4fe71d84869cc129e2721973231ef", + "sha256:f212d4f46b67ff604d11fff7cc62d36b3e8714edf68e44e9760e19be38c03eb0", + "sha256:f7506387e191fe8cdb267f912469a3cccc538ab108471291636a96a54e599556", + "sha256:fac6e277a41163d27dfab5f4ec1f7a83fac94e170665a4a50191b545721c6521", + "sha256:fcd8f556cdc8cfe35e70efb92463082b7f43dd7e547eb071ffc36abc0ca4699b" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==2.1.1" }, "sniffio": { "hashes": [ @@ -96,7 +156,7 @@ "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], "index": "pypi", - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" } }, diff --git a/goat_monitor.py b/goat_monitor.py index 4d7e65c..2e07f5d 100644 --- a/goat_monitor.py +++ b/goat_monitor.py @@ -5,6 +5,7 @@ from typing import List import click import gotify +import numpy as np import toml __version__ = "v0.0.0" @@ -19,19 +20,37 @@ __version__ = "v0.0.0" type=click.Path(path_type=Path), help="Use a .toml configuration file", ) -def wrap(command: List[str], config: Path): +@click.option( + "--retries", + default=0, + type=int, + help="Number of times to try re-running failed command. -1 to retry indefinitely until success", +) +def wrap(command: List[str], config: Path, retries: int): """Wrap an arbitrary command with gotify notifications""" # read settings first with open(config, "r") as f: settings = toml.load(f) + # gotify configuration url = settings["server"] + app_token = settings["app_token"] + with gotify.Gotify(base_url=url, app_token=app_token) as gotify_connection: + # this is just to test configuration. + # I don't know how long a gotify connection will stay up so rather than thinking about it + # I'm just creating a new one whenever I need to send a message + pass - with gotify.Gotify(base_url=url) as gotify_connection: + if retries == -1: + retries = np.inf + if retries < 0: + raise ValueError("Invalid number of retries specified") + + for attempt in range(retries + 1): # run the command result = subprocess.run( - command, + " ".join(command), shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, @@ -41,7 +60,30 @@ def wrap(command: List[str], config: Path): # TODO: print lines real time as the subprocess runs print(result.stdout, end="") - gotify_connection.create_message(message=f'Command "{" ".join(command)}" finished {""}', title="") + if result.returncode: + # failed + title = f"Command failed with exit code {result.returncode}" + if attempt < retries: + title += f" - Retrying (attempt {attempt+1}/{retries})" + else: + title += " - Aborting" + + else: + title = "Command succeeded" + + MAX_LINES = 20 + lines = result.stdout.splitlines() + if len(lines) > MAX_LINES: + lines = lines[-MAX_LINES:] + message = ( + "Command:\n" + + " ".join(command) + + f'\n\nResult{ " (truncated)" if len(lines) > MAX_LINES else ""}:\n' + + "\n".join(lines) + ) + + with gotify.Gotify(base_url=url, app_token=app_token) as gotify_connection: + gotify_connection.create_message(message=message, title=title) return result.returncode