Merge pull request #162 from impl/add-enhanced-networking

Add support for enhanced networking on EC2

CHANGELOG

@@ -1,3 +1,6 @@
+2014-11-23:
+	Noah Fontes:
+		* Add support for enhanced networking on EC2 images
 2014-07-12:
 	Tiago Ilieve:
 		* Fixes #96: AddBackports is now a common task

bootstrapvz/common/tasks/kernel.py

@@ -0,0 +1,26 @@
+from bootstrapvz.base import Task
+from .. import phases
+from ..tasks import packages
+
+
+class AddDKMSPackages(Task):
+	description = 'Adding DKMS and kernel header packages'
+	phase = phases.package_installation
+	successors = [packages.InstallPackages]
+
+	@classmethod
+	def run(cls, info):
+		info.packages.add('dkms')
+		kernel_pkg_arch = {'i386': '686-pae', 'amd64': 'amd64'}[info.manifest.system['architecture']]
+		info.packages.add('linux-headers-' + kernel_pkg_arch)
+
+
+class UpdateInitramfs(Task):
+	description = 'Rebuilding initramfs'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import log_check_call
+	        # Update initramfs (-u) for all currently installed kernel versions (-k all)
+		log_check_call(['chroot', info.root, 'update-initramfs', '-u', '-k', 'all'])

bootstrapvz/providers/ec2/__init__.py

@@ -13,6 +13,7 @@ from bootstrapvz.common.tasks import filesystem
 from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import loopback
+from bootstrapvz.common.tasks import kernel
 
 
 def initialize():
@@ -41,6 +42,7 @@ def validate_manifest(data, validator, error):
 	virtualization = data['provider']['virtualization']
 	backing = data['volume']['backing']
 	partition_type = data['volume']['partitions']['type']
+	enhanced_networking = data['provider']['enhanced_networking'] if 'enhanced_networking' in data['provider'] else None
 
 	if virtualization == 'pvm' and bootloader != 'pvgrub':
 		error('Paravirtualized AMIs only support pvgrub as a bootloader', ['system', 'bootloader'])
@@ -58,6 +60,10 @@ def validate_manifest(data, validator, error):
 		if partition_type != 'none':
 			error('S3 backed AMIs currently only work with unpartitioned volumes', ['system', 'bootloader'])
 
+	if enhanced_networking == 'simple':
+		if virtualization != 'hvm':
+			error('Enhanced networking currently only works with HVM virtualization', ['provider', 'virtualization'])
+
 
 def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
@@ -106,6 +112,11 @@ def resolve_tasks(taskset, manifest):
 		                ])
 		taskset.discard(filesystem.FStab)
 
+	if 'enhanced_networking' in manifest.provider and manifest.provider['enhanced_networking'] == 'simple':
+		taskset.update([kernel.AddDKMSPackages,
+		                tasks.network.InstallEnhancedNetworking,
+		                kernel.UpdateInitramfs])
+
 	taskset.update([filesystem.Format,
 	                volume.Delete,
 	                ])

bootstrapvz/providers/ec2/manifest-schema.yml

@@ -19,6 +19,10 @@ properties:
         enum:
         - pvm
         - hvm
+      enhanced_networking:
+        enum:
+        - none
+        - simple
     required: [virtualization]
   system:
     type: object

bootstrapvz/providers/ec2/tasks/ami.py

@@ -122,4 +122,7 @@ class RegisterAMI(Task):
 			registration_params['kernel_id'] = config_get(akis_path, [info._ec2['region'],
 			                                                          info.manifest.system['architecture']])
 
+		if 'enhanced_networking' in info.manifest.provider and info.manifest.provider['enhanced_networking'] == 'simple':
+			registration_params['sriov_net_support'] = 'simple'
+
 		info._ec2['image'] = info._ec2['connection'].register_image(**registration_params)

bootstrapvz/providers/ec2/tasks/network.py

@@ -1,6 +1,7 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tasks import kernel
 import os.path
 
 
@@ -29,30 +30,39 @@ class AddBuildEssentialPackage(Task):
 
 
 class InstallEnhancedNetworking(Task):
-	description = 'Installing network drivers for SR-IOV support'
+	description = 'Installing enhanced networking kernel driver using DKMS'
-	phase = phases.package_installation
+	phase = phases.system_modification
+	successors = [kernel.UpdateInitramfs]
 
 	@classmethod
 	def run(cls, info):
-		drivers_url = 'http://downloads.sourceforge.net/project/e1000/ixgbevf stable/2.11.3/ixgbevf-2.11.3.tar.gz'
+		version = '2.15.3'
-		archive = os.path.join(info.root, 'tmp', 'ixgbevf-2.11.3.tar.gz')
+		drivers_url = 'http://downloads.sourceforge.net/project/e1000/ixgbevf stable/%s/ixgbevf-%s.tar.gz' % (version, version)
+		archive = os.path.join(info.root, 'tmp', 'ixgbevf-%s.tar.gz' % (version))
+		module_path = os.path.join(info.root, 'usr', 'src', 'ixgbevf-%s' % (version))
 
 		import urllib
 		urllib.urlretrieve(drivers_url, archive)
 
 		from bootstrapvz.common.tools import log_check_call
-		log_check_call('tar', '--ungzip',
+		log_check_call(['tar', '--ungzip',
-		                      '--extract',
+		                       '--extract',
-		                      '--file', archive,
+		                       '--file', archive,
-		                      '--directory', os.path.join(info.root, 'tmp'))
+		                       '--directory', os.path.join(info.root, 'usr', 'src')])
 
-		src_dir = os.path.join('/tmp', os.path.basename(drivers_url), 'src')
+		with open(os.path.join(module_path, 'dkms.conf'), 'w') as dkms_conf:
-		log_check_call(['chroot', info.root,
+			dkms_conf.write("""PACKAGE_NAME="ixgbevf"
-		                'make', '--directory', src_dir])
+PACKAGE_VERSION="%s"
-		log_check_call(['chroot', info.root,
+CLEAN="cd src/; make clean"
-		                'make', 'install',
+MAKE="cd src/; make BUILD_KERNEL=${kernelver}"
-		                        '--directory', src_dir])
+BUILT_MODULE_LOCATION[0]="src/"
+BUILT_MODULE_NAME[0]="ixgbevf"
+DEST_MODULE_LOCATION[0]="/updates"
+DEST_MODULE_NAME[0]="ixgbevf"
+AUTOINSTALL="yes"
+""" % (version))
 
-		ixgbevf_conf_path = os.path.join(info.root, 'etc/modprobe.d/ixgbevf.conf')
+		for task in ['add', 'build', 'install']:
-		with open(ixgbevf_conf_path, 'w') as ixgbevf_conf:
+			# Invoke DKMS task using specified kernel module (-m) and version (-v)
-			ixgbevf_conf.write('options ixgbevf InterruptThrottleRate=1,1,1,1,1,1,1,1')
+			log_check_call(['chroot', info.root,
+			                'dkms', task, '-m', 'ixgbevf', '-v', version])

manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml

@@ -2,6 +2,7 @@
 provider:
   name: ec2
   virtualization: hvm
+  enhanced_networking: simple
   # credentials:
   #   access-key: AFAKEACCESSKEYFORAWS
   #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva

manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml

@@ -2,6 +2,7 @@
 provider:
   name: ec2
   virtualization: hvm
+  enhanced_networking: simple
   # credentials:
   #   access-key: AFAKEACCESSKEYFORAWS
   #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva