From 682b747efc6ba9051562b2005280435a82d40173 Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Tue, 17 Dec 2013 15:51:03 +0000 Subject: [PATCH 1/6] Permit cloud-init to be configured for its metadata sources using debconf. --- .../ec2-ebs-debian-official-amd64-hvm.manifest.json | 3 ++- .../ec2-ebs-debian-official-amd64-pvm.manifest.json | 3 ++- .../ec2-ebs-debian-official-i386-pvm.manifest.json | 3 ++- plugins/cloud_init/__init__.py | 3 ++- plugins/cloud_init/manifest-schema.json | 3 +++ plugins/cloud_init/tasks.py | 13 +++++++++++++ 6 files changed, 24 insertions(+), 4 deletions(-) diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json index 2afe8bd..702d5fb 100644 --- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json @@ -44,7 +44,8 @@ ] }, "cloud_init": { - "username": "admin" + "username": "admin", + "metadata_sources": "Ec2" } } } diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json index 14cd563..ffcf6b1 100644 --- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json @@ -44,7 +44,8 @@ ] }, "cloud_init": { - "username": "admin" + "username": "admin", + "metadata_sources": "Ec2" } } } diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index a3836ea..6428272 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -44,7 +44,8 @@ ] }, "cloud_init": { - "username": "admin" + "username": "admin", + "metadata_sources": "Ec2" } } } diff --git a/plugins/cloud_init/__init__.py b/plugins/cloud_init/__init__.py index ad9d973..b352bf3 100644 --- a/plugins/cloud_init/__init__.py +++ b/plugins/cloud_init/__init__.py @@ -22,9 +22,10 @@ def validate_manifest(data, schema_validate): def resolve_tasks(tasklist, manifest): from tasks import SetUsername + from tasks import SetMetadataSource from providers.ec2.tasks.initd import AddEC2InitScripts from common.tasks import initd - tasklist.add(SetUsername) + tasklist.add(SetUsername, SetMetadataSource) tasklist.remove(AddEC2InitScripts, initd.AddExpandRoot, initd.AdjustExpandRootScript, diff --git a/plugins/cloud_init/manifest-schema.json b/plugins/cloud_init/manifest-schema.json index 599855c..9909339 100644 --- a/plugins/cloud_init/manifest-schema.json +++ b/plugins/cloud_init/manifest-schema.json @@ -11,6 +11,9 @@ "properties": { "username": { "type": "string" + }, + "metadata_sources": { + "type": "string" } }, "required": ["username"] diff --git a/plugins/cloud_init/tasks.py b/plugins/cloud_init/tasks.py index 37883f2..9402032 100644 --- a/plugins/cloud_init/tasks.py +++ b/plugins/cloud_init/tasks.py @@ -1,6 +1,8 @@ from base import Task from common import phases from plugins.packages.tasks import InstallRemotePackages +from common.tasks import apt +from common.tools import log_check_call class SetUsername(Task): @@ -18,3 +20,14 @@ class SetUsername(Task): ' sudo: ALL=(ALL) NOPASSWD:ALL\n' ' shell: /bin/bash').format(username=username) sed_i(cloud_cfg, search, replace) + +class SetMetadataSource(Task): + description = 'Setting metadata source' + phase = phases.system_modification + predecessors = [apt.AptSources] + successors = [apt.AptUpdate] + + def run(self, info): + sources = "cloud-init cloud-init/datasources multiselect " + info.manifest.plugins['cloud_init']['metadata_sources'] + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) + From 02ef8cabb8dafd41859647fd5f5b38a647ac0871 Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Thu, 19 Dec 2013 11:44:34 +0000 Subject: [PATCH 2/6] Support auto-configure of cloud-init data sources, with manual override; support specifying modules to disable. --- ...bs-debian-official-amd64-hvm.manifest.json | 3 +- ...bs-debian-official-amd64-pvm.manifest.json | 3 +- ...ebs-debian-official-i386-pvm.manifest.json | 3 +- plugins/cloud_init/__init__.py | 4 +- plugins/cloud_init/manifest-schema.json | 7 +++ plugins/cloud_init/tasks.py | 46 ++++++++++++++++++- 6 files changed, 60 insertions(+), 6 deletions(-) diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json index 702d5fb..43806d2 100644 --- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json @@ -45,7 +45,8 @@ }, "cloud_init": { "username": "admin", - "metadata_sources": "Ec2" + //"metadata_sources": "Ec2", + "disable_modules": [ "landscape", "byobu", "ssh-import-id" ] } } } diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json index ffcf6b1..b5094e7 100644 --- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json @@ -45,7 +45,8 @@ }, "cloud_init": { "username": "admin", - "metadata_sources": "Ec2" + //"metadata_sources": "Ec2", + "disable_modules": [ "landscape", "byobu", "ssh-import-id" ] } } } diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index 6428272..285d3e5 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -45,7 +45,8 @@ }, "cloud_init": { "username": "admin", - "metadata_sources": "Ec2" + //"metadata_sources": "Ec2", + "disable_modules": [ "landscape", "byobu", "ssh-import-id" ] } } } diff --git a/plugins/cloud_init/__init__.py b/plugins/cloud_init/__init__.py index b352bf3..d80033f 100644 --- a/plugins/cloud_init/__init__.py +++ b/plugins/cloud_init/__init__.py @@ -23,9 +23,11 @@ def validate_manifest(data, schema_validate): def resolve_tasks(tasklist, manifest): from tasks import SetUsername from tasks import SetMetadataSource + from tasks import AutoSetMetadataSource + from tasks import DisableModules from providers.ec2.tasks.initd import AddEC2InitScripts from common.tasks import initd - tasklist.add(SetUsername, SetMetadataSource) + tasklist.add(SetUsername, AutoSetMetadataSource, SetMetadataSource, DisableModules) tasklist.remove(AddEC2InitScripts, initd.AddExpandRoot, initd.AdjustExpandRootScript, diff --git a/plugins/cloud_init/manifest-schema.json b/plugins/cloud_init/manifest-schema.json index 9909339..719716a 100644 --- a/plugins/cloud_init/manifest-schema.json +++ b/plugins/cloud_init/manifest-schema.json @@ -12,6 +12,13 @@ "username": { "type": "string" }, + "disable_modules": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, "metadata_sources": { "type": "string" } diff --git a/plugins/cloud_init/tasks.py b/plugins/cloud_init/tasks.py index 9402032..cea1b6a 100644 --- a/plugins/cloud_init/tasks.py +++ b/plugins/cloud_init/tasks.py @@ -3,6 +3,7 @@ from common import phases from plugins.packages.tasks import InstallRemotePackages from common.tasks import apt from common.tools import log_check_call +import re class SetUsername(Task): @@ -28,6 +29,47 @@ class SetMetadataSource(Task): successors = [apt.AptUpdate] def run(self, info): - sources = "cloud-init cloud-init/datasources multiselect " + info.manifest.plugins['cloud_init']['metadata_sources'] - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) + if "metadata_sources" in info.manifest.plugins['cloud_init']: + sources = "cloud-init cloud-init/datasources multiselect " + info.manifest.plugins['cloud_init']['metadata_sources'] + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) + +class AutoSetMetadataSource(Task): + description = 'Auto-setting metadata source' + phase = phases.system_modification + predecessors = [apt.AptSources] + successors = [SetMetadataSource] + def run(self, info): + sources = "" + if info.manifest.provider == "ec2": + sources = "Ec2" + + if sources: + sources = "cloud-init cloud-init/datasources multiselect " + sources + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) + +class DisableModules(Task): + description = 'Setting cloud.cfg modules' + phase = phases.system_modification + predecessors = [apt.AptUpgrade] + + def run(self, info): + patterns = "" + for pattern in info.manifest.plugins['cloud_init']['disable_modules']: + if patterns != "": + patterns = patterns + "|" + pattern + else: + patterns = "^\s+-\s+(" + pattern + patterns = patterns + ")$" + regex = re.compile(patterns) + + f = open(info.root + "/etc/cloud/cloud.cfg") + lines = f.readlines() + f.close() + + print("Pattern to match is " + patterns + "\n") + f = open(info.root + "/etc/cloud/cloud.cfg", "w") + for line in lines: + if not regex.match(line): + f.write(line) + f.close From e17bc7ec73599095a747a5a58af23a5441d71bdc Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Thu, 19 Dec 2013 14:21:33 +0000 Subject: [PATCH 3/6] Adjust run ordering of cloud-init config. --- plugins/cloud_init/tasks.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/plugins/cloud_init/tasks.py b/plugins/cloud_init/tasks.py index cea1b6a..fe6df1b 100644 --- a/plugins/cloud_init/tasks.py +++ b/plugins/cloud_init/tasks.py @@ -45,6 +45,7 @@ class AutoSetMetadataSource(Task): sources = "Ec2" if sources: + print ("Setting metadata source to " + sources) sources = "cloud-init cloud-init/datasources multiselect " + sources log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) @@ -54,22 +55,22 @@ class DisableModules(Task): predecessors = [apt.AptUpgrade] def run(self, info): - patterns = "" - for pattern in info.manifest.plugins['cloud_init']['disable_modules']: - if patterns != "": - patterns = patterns + "|" + pattern - else: - patterns = "^\s+-\s+(" + pattern - patterns = patterns + ")$" - regex = re.compile(patterns) + if 'disable_modules' in info.manifest.plugins['cloud_init']: + patterns = "" + for pattern in info.manifest.plugins['cloud_init']['disable_modules']: + if patterns != "": + patterns = patterns + "|" + pattern + else: + patterns = "^\s+-\s+(" + pattern + patterns = patterns + ")$" + regex = re.compile(patterns) - f = open(info.root + "/etc/cloud/cloud.cfg") - lines = f.readlines() - f.close() + f = open(info.root + "/etc/cloud/cloud.cfg") + lines = f.readlines() + f.close() - print("Pattern to match is " + patterns + "\n") - f = open(info.root + "/etc/cloud/cloud.cfg", "w") - for line in lines: + f = open(info.root + "/etc/cloud/cloud.cfg", "w") + for line in lines: if not regex.match(line): f.write(line) - f.close + f.close From d9dacc5d41c35f98d61ae3fe5e5e26d60d7510f5 Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Fri, 20 Dec 2013 15:04:22 +0000 Subject: [PATCH 4/6] Simplify the description, make the AMI name have the year and the fact it is ebs. --- manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json | 4 ++-- manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json | 4 ++-- manifests/ec2-ebs-debian-official-i386-pvm.manifest.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json index 43806d2..45c726c 100644 --- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" + "name": "debian-{release}-{architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {release} {architecture}" }, "system": { "release": "wheezy", diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json index b5094e7..9631562 100644 --- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" + "name": "debian-{release}-{architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {release} {architecture}" }, "system": { "release": "wheezy", diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index 285d3e5..e44cfe6 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" + "name": "debian-{release}-{architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {release} {architecture}" }, "system": { "release": "wheezy", From 7e3527d2afa012a9ff4f611ebcaaa6d3c401a1ae Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Sat, 21 Dec 2013 12:26:37 +0000 Subject: [PATCH 5/6] PIP8 ify the latest updates. --- plugins/cloud_init/tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/cloud_init/tasks.py b/plugins/cloud_init/tasks.py index fe6df1b..df8f917 100644 --- a/plugins/cloud_init/tasks.py +++ b/plugins/cloud_init/tasks.py @@ -22,6 +22,7 @@ class SetUsername(Task): ' shell: /bin/bash').format(username=username) sed_i(cloud_cfg, search, replace) + class SetMetadataSource(Task): description = 'Setting metadata source' phase = phases.system_modification @@ -31,7 +32,7 @@ class SetMetadataSource(Task): def run(self, info): if "metadata_sources" in info.manifest.plugins['cloud_init']: sources = "cloud-init cloud-init/datasources multiselect " + info.manifest.plugins['cloud_init']['metadata_sources'] - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections'], sources) class AutoSetMetadataSource(Task): @@ -39,6 +40,7 @@ class AutoSetMetadataSource(Task): phase = phases.system_modification predecessors = [apt.AptSources] successors = [SetMetadataSource] + def run(self, info): sources = "" if info.manifest.provider == "ec2": @@ -47,7 +49,8 @@ class AutoSetMetadataSource(Task): if sources: print ("Setting metadata source to " + sources) sources = "cloud-init cloud-init/datasources multiselect " + sources - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections' ], sources) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections'], sources) + class DisableModules(Task): description = 'Setting cloud.cfg modules' From 39f5e815b82e30058ae96922ce26fc6fa81449b1 Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Wed, 25 Dec 2013 12:47:17 +0000 Subject: [PATCH 6/6] Process cloud config tasks only if they are configured. --- plugins/cloud_init/__init__.py | 11 ++++++++++- plugins/cloud_init/tasks.py | 10 +++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/plugins/cloud_init/__init__.py b/plugins/cloud_init/__init__.py index d80033f..df3a11c 100644 --- a/plugins/cloud_init/__init__.py +++ b/plugins/cloud_init/__init__.py @@ -27,7 +27,16 @@ def resolve_tasks(tasklist, manifest): from tasks import DisableModules from providers.ec2.tasks.initd import AddEC2InitScripts from common.tasks import initd - tasklist.add(SetUsername, AutoSetMetadataSource, SetMetadataSource, DisableModules) + + options = manifest.plugins['cloud_init'] + tasklist.add(AutoSetMetadataSource) + if 'username' in options: + tasklist.add(SetUsername) + if 'disable_modules' in options: + tasklist.add(DisableModules) + if 'metadata_sources' in options: + tasklist.add(SetMetadataSource) + tasklist.remove(AddEC2InitScripts, initd.AddExpandRoot, initd.AdjustExpandRootScript, diff --git a/plugins/cloud_init/tasks.py b/plugins/cloud_init/tasks.py index df8f917..5613a56 100644 --- a/plugins/cloud_init/tasks.py +++ b/plugins/cloud_init/tasks.py @@ -68,9 +68,13 @@ class DisableModules(Task): patterns = patterns + ")$" regex = re.compile(patterns) - f = open(info.root + "/etc/cloud/cloud.cfg") - lines = f.readlines() - f.close() + try: + f = open(info.root + "/etc/cloud/cloud.cfg") + lines = f.readlines() + f.close() + except: + print "Cannot read cloud.cfg" + return -1 f = open(info.root + "/etc/cloud/cloud.cfg", "w") for line in lines: