GPT support for both extlinux and grub

base/bootstrapinfo.py

@@ -12,7 +12,7 @@ class BootstrapInformation(object):
 		self.workspace = os.path.join(manifest.bootstrapper['workspace'], self.run_id)
 
 		from fs import load_volume
-		self.volume = load_volume(self.manifest.volume)
+		self.volume = load_volume(self.manifest.volume, manifest.system['bootloader'])
 
 		self.apt_mirror = self.manifest.packages.get('mirror', 'http://http.debian.net/debian')
 

base/fs/__init__.py

@@ -1,6 +1,6 @@
 
 
-def load_volume(data):
+def load_volume(data, bootloader):
 	from common.fs.loopbackvolume import LoopbackVolume
 	from providers.ec2.ebsvolume import EBSVolume
 	from common.fs.virtualdiskimage import VirtualDiskImage
@@ -12,7 +12,7 @@ def load_volume(data):
 	                  'gpt': GPTPartitionMap,
 	                  'msdos': MSDOSPartitionMap,
 	                  }
-	partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'])
+	partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'], bootloader)
 	volume_backings = {'raw': LoopbackVolume,
 	                   's3':  LoopbackVolume,
 	                   'vdi': VirtualDiskImage,

base/fs/partitionmaps/abstract.py

@@ -14,7 +14,7 @@ class AbstractPartitionMap(FSMProxy):
 	          {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'},
 	          ]
 
-	def __init__(self):
+	def __init__(self, bootloader):
 		cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}}
 		super(AbstractPartitionMap, self).__init__(cfg)
 
@@ -22,7 +22,7 @@ class AbstractPartitionMap(FSMProxy):
 		return self.fsm.current == 'mapped'
 
 	def get_total_size(self):
-		return sum(p.size for p in self.partitions)
+		return self.partitions[-1].get_end()
 
 	def create(self, volume):
 		self.fsm.create(volume=volume)

base/fs/partitionmaps/gpt.py

@@ -6,13 +6,20 @@ from common.tools import log_check_call
 
 class GPTPartitionMap(AbstractPartitionMap):
 
-	def __init__(self, data):
+	def __init__(self, data, bootloader):
 		self.partitions = []
 
 		def last_partition():
 			return self.partitions[-1] if len(self.partitions) > 0 else None
+
+		if bootloader == 'grub':
+			from ..partitions.unformatted import UnformattedPartition
+			self.grub_boot = UnformattedPartition(2, last_partition())
+			self.grub_boot.flags.append('bios_grub')
+			self.partitions.append(self.grub_boot)
+
 		if 'boot' in data:
-			self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None)
+			self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', last_partition())
 			self.partitions.append(self.boot)
 		if 'swap' in data:
 			self.swap = GPTSwapPartition(data['swap']['size'], last_partition())
@@ -20,7 +27,11 @@ class GPTPartitionMap(AbstractPartitionMap):
 		self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', last_partition())
 		self.partitions.append(self.root)
 
-		super(GPTPartitionMap, self).__init__()
+		# getattr(self, 'boot', self.root).flags.append('boot')
+		if bootloader == 'extlinux':
+			getattr(self, 'boot', self.root).flags.append('legacy_boot')
+
+		super(GPTPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):
 		volume = event.volume
@@ -28,9 +39,3 @@ class GPTPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'gpt'])
 		for partition in self.partitions:
 			partition.create(volume)
-
-		boot_idx = getattr(self, 'boot', self.root).get_index()
-		log_check_call(['/sbin/parted', '--script', volume.device_path,
-		                '--', 'set ' + str(boot_idx) + ' boot on'])
-		log_check_call(['/sbin/parted', '--script', volume.device_path,
-		                '--', 'set ' + str(boot_idx) + ' bios_grub on'])

base/fs/partitionmaps/msdos.py

@@ -6,7 +6,7 @@ from common.tools import log_check_call
 
 class MSDOSPartitionMap(AbstractPartitionMap):
 
-	def __init__(self, data):
+	def __init__(self, data, bootloader):
 		self.partitions = []
 
 		def last_partition():
@@ -20,7 +20,11 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		self.root = MSDOSPartition(data['root']['size'], data['root']['filesystem'], last_partition())
 		self.partitions.append(self.root)
 
-		super(MSDOSPartitionMap, self).__init__()
+		getattr(self, 'boot', self.root).flags.append('boot')
+
+		if bootloader == 'grub':
+			self.partitions[0].offset = 2
+		super(MSDOSPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):
 		volume = event.volume
@@ -28,7 +32,3 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'msdos'])
 		for partition in self.partitions:
 			partition.create(volume)
-
-		boot_idx = getattr(self, 'boot', self.root).get_index()
-		log_check_call(['/sbin/parted', '--script', volume.device_path,
-		                '--', 'set ' + str(boot_idx) + ' boot on'])

base/fs/partitionmaps/none.py

@@ -3,7 +3,7 @@ from ..partitions.single import SinglePartition
 
 class NoPartitions(object):
 
-	def __init__(self, data):
+	def __init__(self, data, bootloader):
 		root = data['root']
 		self.root = SinglePartition(root['size'], root['filesystem'])
 		self.partitions = [self.root]

base/fs/partitions/abstract.py

@@ -1,4 +1,5 @@
 from abc import ABCMeta
+from abc import abstractmethod
 import os.path
 from common.tools import log_check_call
 from common.fsm_proxy import FSMProxy
@@ -23,7 +24,7 @@ class AbstractPartition(FSMProxy):
 		def mount(self, prefix):
 			mount_dir = os.path.join(prefix, self.destination)
 			if isinstance(self.source, AbstractPartition):
-				self.source.mount(mount_dir)
+				self.source.mount(destination=mount_dir)
 			else:
 				log_check_call(['/bin/mount'] + self.opts + [self.source, mount_dir])
 			self.mount_dir = mount_dir
@@ -48,13 +49,17 @@ class AbstractPartition(FSMProxy):
 		[uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path])
 		return uuid
 
+	@abstractmethod
+	def get_start(self):
+		pass
+
+	def get_end(self):
+		return self.get_start() + self.size
+
 	def _before_format(self, e):
 		mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem)
 		log_check_call([mkfs, self.device_path])
 
-	def mount(self, destination):
-		self.fsm.mount(destination=destination)
-
 	def _before_mount(self, e):
 		log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination])
 		self.mount_dir = e.destination

base/fs/partitions/base.py

@@ -16,6 +16,8 @@ class BasePartition(AbstractPartition):
 
 	def __init__(self, size, filesystem, previous):
 		self.previous = previous
+		self.offset = 0
+		self.flags = []
 		super(BasePartition, self).__init__(size, filesystem)
 
 	def create(self, volume):
@@ -29,13 +31,26 @@ class BasePartition(AbstractPartition):
 
 	def get_start(self):
 		if self.previous is None:
-			return 0
+			return self.offset
 		else:
-			return self.previous.get_start() + self.previous.size
+			return self.previous.get_end() + self.offset
 
 	def map(self, device_path):
 		self.fsm.map(device_path=device_path)
 
+	def _before_create(self, e):
+		from common.tools import log_check_call
+		create_command = ('mkpart primary {start}MiB {end}MiB'
+		                  .format(start=str(self.get_start()),
+		                          end=str(self.get_end())))
+		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
+		                '--', create_command])
+
+		for flag in self.flags:
+			log_check_call(['/sbin/parted', '--script', e.volume.device_path,
+			                '--', ('set {idx} {flag} on'
+			                       .format(idx=str(self.get_index()), flag=flag))])
+
 	def _before_map(self, e):
 		self.device_path = e.device_path
 

base/fs/partitions/gpt.py

@@ -9,16 +9,10 @@ class GPTPartition(BasePartition):
 		super(GPTPartition, self).__init__(size, filesystem, previous)
 
 	def _before_create(self, e):
-		start = self.get_start()
-		create_command = ('mkpart primary {start}MiB {end}MiB'
-		                  .format(start=str(start),
-		                          end=str(start + self.size)))
-		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
-		                '--', create_command])
-
+		super(GPTPartition, self)._before_create(e)
 		# partition name only works for gpt, for msdos that becomes the part-type (primary, extended, logical)
 		name_command = ('name {idx} {name}'
 		                .format(idx=self.get_index(),
 		                        name=self.name))
-		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
+		log_check_call(['/sbin/parted', '--script', e.volume.device_path,
 		                '--', name_command])

base/fs/partitions/msdos.py

@@ -1,19 +1,5 @@
-from common.tools import log_check_call
 from base import BasePartition
 
 
 class MSDOSPartition(BasePartition):
-
-	def get_start(self):
-		if self.previous is None:
-			return 2  # Post-MBR gap for embedding grub
-		else:
-			return self.previous.get_start() + self.previous.size
-
-	def _before_create(self, e):
-		start = self.get_start()
-		parted_command = ('mkpart primary {start}MiB {end}MiB'
-		                  .format(start=str(start),
-		                          end=str(start + self.size)))
-		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
-		                '--', parted_command])
+	pass

base/fs/partitions/single.py

@@ -2,4 +2,6 @@ from abstract import AbstractPartition
 
 
 class SinglePartition(AbstractPartition):
-		pass
+
+	def get_start(self):
+		return 0

base/fs/partitions/unformatted.py

@@ -0,0 +1,12 @@
+from base import BasePartition
+
+
+class UnformattedPartition(BasePartition):
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'},
+	          {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'},
+	          {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'},
+	          ]
+
+	def __init__(self, size, previous):
+		super(UnformattedPartition, self).__init__(size, None, previous)

base/fs/volume.py

@@ -3,7 +3,6 @@ from common.fsm_proxy import FSMProxy
 from common.tools import log_check_call
 from exceptions import VolumeError
 from partitionmaps.none import NoPartitions
-from partitionmaps.mbr import MSDOSPartitionMap
 
 
 class Volume(FSMProxy):
@@ -23,8 +22,6 @@ class Volume(FSMProxy):
 		self.real_device_path = None
 		self.partition_map = partition_map
 		self.size = self.partition_map.get_total_size()
-		if isinstance(self.partition_map, MSDOSPartitionMap):
-			self.size += 2  # Post-MBR gap for embedding bootloader
 
 		callbacks = {'onbeforedetach': self._check_blocking}
 		if isinstance(self.partition_map, NoPartitions):

common/fs/__init__.py

@@ -28,5 +28,5 @@ def remount(volume, fn):
 		p_map.map(volume)
 	else:
 		result = fn()
-	p_map.root.mount(root_dir)
+	p_map.root.mount(destination=root_dir)
 	return result

common/tasks/boot.py

@@ -2,6 +2,7 @@ from base import Task
 from common import phases
 from common.tasks import apt
 from common.tasks import filesystem
+from base.fs import partitionmaps
 import os.path
 
 
@@ -56,19 +57,17 @@ class InstallGrub(Task):
 		boot_dir = os.path.join(info.root, 'boot')
 		grub_dir = os.path.join(boot_dir, 'grub')
 
-		from base.fs.partitionmaps.none import NoPartitions
-		from base.fs.partitionmaps.gpt import GPTPartitionMap
 		from common.fs import remount
 		p_map = info.volume.partition_map
 
 		def link_fn():
 			info.volume.link_dm_node()
-			if isinstance(p_map, NoPartitions):
+			if isinstance(p_map, partitionmaps.none.NoPartitions):
 				p_map.root.device_path = info.volume.device_path
 
 		def unlink_fn():
 			info.volume.unlink_dm_node()
-			if isinstance(p_map, NoPartitions):
+			if isinstance(p_map, partitionmaps.none.NoPartitions):
 				p_map.root.device_path = info.volume.device_path
 
 		# GRUB cannot deal with installing to loopback devices
@@ -80,11 +79,11 @@ class InstallGrub(Task):
 			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
 			device_map_path = os.path.join(grub_dir, 'device.map')
 			partition_prefix = 'msdos'
-			if isinstance(p_map, GPTPartitionMap):
+			if isinstance(p_map, partitionmaps.gpt.GPTPartitionMap):
 				partition_prefix = 'gpt'
 			with open(device_map_path, 'w') as device_map:
 				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
-				if not isinstance(p_map, NoPartitions):
+				if not isinstance(p_map, partitionmaps.none.NoPartitions):
 					for idx, partition in enumerate(info.volume.partition_map.partitions):
 						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
 						                 .format(device_path=partition.device_path,
@@ -93,10 +92,7 @@ class InstallGrub(Task):
 
 			# Install grub
 			log_check_call(['/usr/sbin/chroot', info.root,
-			                '/usr/sbin/grub-install',
-			                # '--root-directory=' + info.root,
-			                # '--boot-directory=' + boot_dir,
-			                device_path])
+			                '/usr/sbin/grub-install', device_path])
 			log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
 		except Exception as e:
 			if isinstance(info.volume, LoopbackVolume):
@@ -115,6 +111,8 @@ class AddExtlinuxPackage(Task):
 	@classmethod
 	def run(cls, info):
 		info.packages.add('extlinux')
+		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
+			info.packages.add('syslinux-common')
 
 
 class InstallExtLinux(Task):
@@ -125,7 +123,16 @@ class InstallExtLinux(Task):
 	@classmethod
 	def run(cls, info):
 		from common.tools import log_check_call
+		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
+			bootloader = '/usr/lib/syslinux/gptmbr.bin'
+		else:
+			bootloader = '/usr/lib/extlinux/mbr.bin'
 		log_check_call(['/usr/sbin/chroot', info.root,
-		                '/usr/sbin/extlinux-install', info.volume.device_path])
+		                '/bin/dd', 'bs=440', 'count=1',
+		                'if=' + bootloader,
+		                'of=' + info.volume.device_path])
+		log_check_call(['/usr/sbin/chroot', info.root,
+		                '/usr/bin/extlinux',
+		                '--install', '/boot/extlinux'])
 		log_check_call(['/usr/sbin/chroot', info.root,
 		                '/usr/sbin/extlinux-update'])

common/tasks/filesystem.py

@@ -12,8 +12,10 @@ class Format(Task):
 
 	@classmethod
 	def run(cls, info):
+		from base.fs.partitions.unformatted import UnformattedPartition
 		for partition in info.volume.partition_map.partitions:
-			partition.format()
+			if not isinstance(partition, UnformattedPartition):
+				partition.format()
 
 
 class TuneVolumeFS(Task):
@@ -23,11 +25,13 @@ class TuneVolumeFS(Task):
 
 	@classmethod
 	def run(cls, info):
+		from base.fs.partitions.unformatted import UnformattedPartition
 		import re
 		# Disable the time based filesystem check
 		for partition in info.volume.partition_map.partitions:
-			if re.match('^ext[2-4]$', partition.filesystem) is not None:
-				log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path])
+			if not isinstance(partition, UnformattedPartition):
+				if re.match('^ext[2-4]$', partition.filesystem) is not None:
+					log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path])
 
 
 class AddXFSProgs(Task):
@@ -58,7 +62,7 @@ class MountRoot(Task):
 
 	@classmethod
 	def run(cls, info):
-		info.volume.partition_map.root.mount(info.root)
+		info.volume.partition_map.root.mount(destination=info.root)
 
 
 class CreateBootMountDir(Task):

providers/virtualbox/manifest-schema.json

@@ -30,7 +30,7 @@
 				"partitions": {
 					"type": "object",
 					"properties": {
-						"type": { "enum": ["none", "msdos"] }
+						"type": { "enum": ["none", "msdos", "gpt"] }
 					}
 				}
 			},