From c6fe2d592ed40ee81988ce5e160959206e4a15a0 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Tue, 5 Jun 2018 19:21:07 -0700 Subject: [PATCH 1/4] Add keep_by_hours option To be more adaptive to user needs, keep_by_hours option works as the inverse of the "delete_by_hours" option. "keep_by_hours" will keep any tag newer than the specified number of hours. --- registry.py | 63 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/registry.py b/registry.py index c44f2f5..0218528 100755 --- a/registry.py +++ b/registry.py @@ -58,7 +58,7 @@ class Requests: print('[debug][registry][request]: {0} {1}'.format(method, url)) if 'Authorization' in kwargs['headers']: print('[debug][registry][request]: Authorization header:') - + token_parsed = kwargs['headers']['Authorization'].split('.') pprint.pprint(ast.literal_eval(decode_base64(token_parsed[0]))) pprint.pprint(ast.literal_eval(decode_base64(token_parsed[1]))) @@ -67,12 +67,12 @@ class Requests: if str(res.status_code)[0] == '2': if DEBUG: print("[debug][registry] accepted") return (res, kwargs['headers']['Authorization']) - + if res.status_code == 401: if DEBUG: print("[debug][registry] Access denied. Refreshing token...") oauth = www_authenticate.parse(res.headers['Www-Authenticate']) - if DEBUG: + if DEBUG: print('[debug][auth][answer] Auth header:') pprint.pprint(oauth['bearer']) @@ -81,11 +81,11 @@ class Requests: oauth['bearer']['service'], oauth['bearer']['scope']) - if DEBUG: + if DEBUG: print('[debug][auth][request] Refreshing auth token: POST {0}'.format(request_url)) - + try_oauth = requests.post(request_url, auth=auth, **kwargs) - + try: token = ast.literal_eval(try_oauth._content)['token'] except SyntaxError: @@ -101,7 +101,7 @@ class Requests: kwargs['headers']['Authorization'] = 'Bearer {0}'.format(token) else: return (res, kwargs['headers']['Authorization']) - + res = requests.request(method, url, **kwargs) return (res, kwargs['headers']['Authorization']) @@ -151,11 +151,11 @@ def get_auth_schemes(r,path): - www-authenticate: basic - www-authenticate: bearer """ - + if DEBUG: print("[debug][funcname]: get_auth_schemes()") try_oauth = requests.head('{0}{1}'.format(r.hostname,path), verify=not r.no_validate_ssl) - + if 'Www-Authenticate' in try_oauth.headers: oauth = www_authenticate.parse(try_oauth.headers['Www-Authenticate']) if DEBUG: @@ -501,7 +501,14 @@ for more detail on garbage collection read here: parser.add_argument( '--delete-by-hours', - help=('Will delete all tags that older than specified hours. Be careful!'), + help=('Will delete all tags that are older than specified hours. Be careful!'), + default=False, + nargs='?', + metavar='Hours') + + parser.add_argument( + '--keep-by-hours', + help=('Will keep all tags that are newer than specified hours.'), default=False, nargs='?', metavar='Hours') @@ -605,11 +612,35 @@ def delete_tags_by_age(registry, image_name, dry_run, hours, tags_to_keep): delete_tags(registry, image_name, dry_run, tags_to_delete, tags_to_keep) +def get_newer_tags(registry, image_name, hours, tags_list): + newer_tags = [] + print('---------------------------------') + for tag in tags_list: + image_config = registry.get_tag_config(image_name, tag) + + if image_config == []: + print("tag not found") + continue + + image_age = registry.get_image_age(image_name, image_config) + + if image_age == []: + print("timestamp not found") + continue + + if dt.strptime(image_age[:-4], "%Y-%m-%dT%H:%M:%S.%f") >= dt.now() - timedelta(hours=int(hours)): + print("Keeping tag: {0} timestamp: {1}".format( + tag, image_age)) + newer_tags.append(tag) + + return newer_tags + + def main_loop(args): global DEBUG DEBUG = True if args.debug else False - + keep_last_versions = int(args.num) if args.no_validate_ssl: @@ -644,9 +675,9 @@ def main_loop(args): registry = Registry.create(args.host, args.login, args.no_validate_ssl, args.digest_method) - + registry.auth_schemes = get_auth_schemes(registry,'/v2/_catalog') - + if args.delete: print("Will delete all but {0} last tags".format(keep_last_versions)) @@ -682,10 +713,14 @@ def main_loop(args): layer['blobSum'])) # add tags to "tags_to_keep" list, if we have regexp "tags_to_keep" - # entries: + # entries or a number of hours for "keep_by_hours": keep_tags = [] if args.keep_tags_like: keep_tags.extend(get_tags_like(args.keep_tags_like, tags_list)) + if args.keep_by_hours: + keep_tags.extend(get_newer_tags(registry, image_name, + args.keep_by_hours, tags_list)) + keep_tags = list(set(keep_tags)) # Eliminate duplicates # delete tags if told so if args.delete or args.delete_all: From 5e37104f249b861be2aee9dff522838764c65a86 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Fri, 8 Jun 2018 10:55:25 -0700 Subject: [PATCH 2/4] Add some keep_by_hours/get_newer_tags tests --- test.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index a6b584a..954136c 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,6 @@ import unittest from registry import Registry, Requests, get_tags, parse_args, \ - delete_tags, delete_tags_by_age, get_error_explanation + delete_tags, delete_tags_by_age, get_error_explanation, get_newer_tags from mock import MagicMock, patch import requests @@ -150,7 +150,7 @@ class TestRegistrySend(unittest.TestCase): headers=self.registry.HEADERS, verify=True) -class TestGetrrorExplanation(unittest.TestCase): +class TestGetErrorExplanation(unittest.TestCase): def test_get_tag_digest_404(self): self.assertEqual(get_error_explanation("delete_tag", "405"), 'You might want to set REGISTRY_STORAGE_DELETE_ENABLED: "true" in your registry') @@ -681,6 +681,39 @@ class TestDeleteTagsByAge(unittest.TestCase): self.registry, "imagename", True, ["image"], ["latest"]) +class TestGetNewerTags(unittest.TestCase): + + def setUp(self): + self.registry = Registry() + self.registry.http = MockRequests() + + self.get_tag_config_mock = MagicMock(return_value={'mediaType': 'application/vnd.docker.container.image.v1+json', 'size': 12953, + 'digest': 'sha256:8d71dfbf239c0015ad66993d55d3954cee2d52d86f829fdff9ccfb9f23b75aa8'}) + self.registry.get_tag_config = self.get_tag_config_mock + self.get_image_age_mock = MagicMock( + return_value="2017-12-27T12:47:33.511765448Z") + self.registry.get_image_age = self.get_image_age_mock + self.list_tags_mock = MagicMock(return_value=["image"]) + self.registry.list_tags = self.list_tags_mock + self.get_tag_digest_mock = MagicMock() + self.registry.get_tag_digest = self.get_tag_digest_mock + self.registry.http = MockRequests() + self.registry.hostname = "http://testdomain.com" + self.registry.http.reset_return_value(200, "MOCK_DIGEST") + + def test_keep_tags_by_age_no_keep(self): + self.assertEqual( + get_newer_tags(self.registry, "imagename", 23, ["latest"]), + [] + ) + + def test_keep_tags_by_age_keep(self): + self.assertEqual( + get_newer_tags(self.registry, "imagename", 24, ["latest"]), + ["latest"] + ) + + class TestArgParser(unittest.TestCase): def test_no_args(self): @@ -700,6 +733,7 @@ class TestArgParser(unittest.TestCase): "--delete-all", "--layers", "--delete-by-hours", "24", + "--keep-by-hours", "24", "--digest-method", "GET"] args = parse_args(args_list) self.assertTrue(args.delete) @@ -714,6 +748,7 @@ class TestArgParser(unittest.TestCase): self.assertEqual(args.host, "hostname") self.assertEqual(args.keep_tags, ["keep1", "keep2"]) self.assertEqual(args.delete_by_hours, "24") + self.assertEqual(args.keep_by_hours, "24") self.assertEqual(args.digest_method, "GET") def test_default_args(self): From e05c6a6f76689af35f235048a7c9258c83b49b74 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Fri, 8 Jun 2018 15:56:41 -0700 Subject: [PATCH 3/4] Update README.md with keep-by-hours help --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8baf444..b96f7f6 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ registry.py is a script for easy manipulation of docker-registry from command li * [Installation](#installation) * [Docker image](#docker-image) * [Python script](#python-script) -* [Listing images](#listing-images) +* [Listing images](#listing-images) * [Username and password](#username-and-password) * [Deleting images](#deleting-images) * [Disable ssl verification](#disable-ssl-verification) @@ -113,7 +113,7 @@ The following command would delete all tags containing "snapshot-" and beginning As one manifest may be referenced by more than one tag, you may add tags, whose manifests should NOT be deleted. A tag that would otherwise be deleted, but whose manifest references one of those "kept" tags, is spared for deletion. -In the following case, all tags beginning with "snapshot-" will be deleted, safe those whose manifest point to "stable" or "latest" +In the following case, all tags beginning with "snapshot-" will be deleted, save those whose manifest point to "stable" or "latest": ``` registry.py -l user:pass -r https://example.com:5000 --delete --tags-like "snapshot-" --keep-tags "stable" "latest" @@ -135,6 +135,11 @@ Delete all tags by age in hours for the particular image (e.g. older than 24 hou ``` registry.py -r https://example.com:5000 -i api-docs-origin/master --dry-run --delete-by-hours 24 --keep-tags c59c02c25f023263fd4b5d43fc1ff653f08b3d4x --keep-tags-like late ``` + +Note that deleting by age will not prevent more recent tags from being deleted if there are more than 10 (or specified `--num` value). In order to keep all tags within a designated period, use the `--keep-by-hours` flag: +``` + registry.py -r https://example.com:5000 --dry-run --delete --keep-by-hours 72 --keep-tags-like latest +``` ## Disable ssl verification If you are using docker registry with a self signed ssl certificate, you can disable ssl verification: From 2795dc5d918c070e6419a16f1e5a666208a088d3 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Fri, 8 Jun 2018 16:18:02 -0700 Subject: [PATCH 4/4] Standardize arg references to plaintext --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b96f7f6..fd706e0 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ In the following case, all tags beginning with "snapshot-" will be deleted, save ``` registry.py -l user:pass -r https://example.com:5000 --delete --tags-like "snapshot-" --keep-tags "stable" "latest" ``` -The last parameter is also available as regexp option with "--keep-tags-like". +The last parameter is also available as regexp option with `--keep-tags-like`. Delete all tags for particular image (e.g. delete all ubuntu tags): @@ -131,7 +131,7 @@ Delete all tags for all images (do you really want to do it?): registry.py -l user:pass -r https://example.com:5000 --delete-all --dry-run ``` -Delete all tags by age in hours for the particular image (e.g. older than 24 hours, with --keep-tags and --keep-tags-like options, --dry-run for safe). +Delete all tags by age in hours for the particular image (e.g. older than 24 hours, with `--keep-tags` and `--keep-tags-like` options, `--dry-run` for safe). ``` registry.py -r https://example.com:5000 -i api-docs-origin/master --dry-run --delete-by-hours 24 --keep-tags c59c02c25f023263fd4b5d43fc1ff653f08b3d4x --keep-tags-like late ``` @@ -149,7 +149,7 @@ If you are using docker registry with a self signed ssl certificate, you can dis ## Nexus docker registry -Add --digest-method flag +Add `--digest-method` flag ``` registry.py -l user:pass -r https://example.com:5000 --digest-method GET