diff --git a/docs/_static/graph.json b/docs/_static/graph.json new file mode 100644 index 0000000..7f2411d --- /dev/null +++ b/docs/_static/graph.json @@ -0,0 +1 @@ +{"phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}], "modules": [{"name": "common.tasks.apt"}, {"name": "common.tasks.boot"}, {"name": "common.tasks.bootstrap"}, {"name": "common.tasks.cleanup"}, {"name": "common.tasks.development"}, {"name": "common.tasks.filesystem"}, {"name": "common.tasks.host"}, {"name": "common.tasks.initd"}, {"name": "common.tasks.locale"}, {"name": "common.tasks.loopback"}, {"name": "common.tasks.network"}, {"name": "common.tasks.packages"}, {"name": "common.tasks.partitioning"}, {"name": "common.tasks.security"}, {"name": "common.tasks.volume"}, {"name": "common.tasks.workspace"}, {"name": "plugins.admin_user.tasks"}, {"name": "plugins.build_metadata.tasks"}, {"name": "plugins.cloud_init.tasks"}, {"name": "plugins.image_commands.tasks"}, {"name": "plugins.minimize_size.tasks"}, {"name": "plugins.opennebula.tasks"}, {"name": "plugins.prebootstrapped.tasks"}, {"name": "plugins.puppet.tasks"}, {"name": "plugins.root_password.tasks"}, {"name": "plugins.unattended_upgrades.tasks"}, {"name": "plugins.vagrant.tasks"}, {"name": "providers.ec2.tasks.ami"}, {"name": "providers.ec2.tasks.boot"}, {"name": "providers.ec2.tasks.connection"}, {"name": "providers.ec2.tasks.ebs"}, {"name": "providers.ec2.tasks.filesystem"}, {"name": "providers.ec2.tasks.host"}, {"name": "providers.ec2.tasks.initd"}, {"name": "providers.ec2.tasks.network"}, {"name": "providers.ec2.tasks.packages"}, {"name": "providers.kvm.tasks.packages"}, {"name": "providers.kvm.tasks.virtio"}, {"name": "providers.virtualbox.tasks.guest_additions"}, {"name": "providers.virtualbox.tasks.packages"}], "nodes": [{"phase": 0, "name": "AddDefaultSources", "module": 0}, {"phase": 0, "name": "AddManifestSources", "module": 0}, {"phase": 7, "name": "AptClean", "module": 0}, {"phase": 5, "name": "AptUpdate", "module": 0}, {"phase": 5, "name": "AptUpgrade", "module": 0}, {"phase": 5, "name": "DisableDaemonAutostart", "module": 0}, {"phase": 7, "name": "EnableDaemonAutostart", "module": 0}, {"phase": 5, "name": "InstallTrustedKeys", "module": 0}, {"phase": 7, "name": "PurgeUnusedPackages", "module": 0}, {"phase": 5, "name": "WriteSources", "module": 0}, {"phase": 0, "name": "AddExtlinuxPackage", "module": 1}, {"phase": 0, "name": "AddGrubPackage", "module": 1}, {"phase": 6, "name": "BlackListModules", "module": 1}, {"phase": 6, "name": "DisableGetTTYs", "module": 1}, {"phase": 6, "name": "InstallExtLinux", "module": 1}, {"phase": 6, "name": "InstallGrub", "module": 1}, {"phase": 4, "name": "Bootstrap", "module": 2}, {"phase": 4, "name": "MakeTarball", "module": 2}, {"phase": 7, "name": "CleanTMP", "module": 3}, {"phase": 7, "name": "ClearMOTD", "module": 3}, {"phase": 7, "name": "ShredHostkeys", "module": 3}, {"phase": 10, "name": "TriggerRollback", "module": 4}, {"phase": 0, "name": "AddXFSProgs", "module": 5}, {"phase": 3, "name": "CreateBootMountDir", "module": 5}, {"phase": 3, "name": "CreateMountDir", "module": 5}, {"phase": 8, "name": "DeleteMountDir", "module": 5}, {"phase": 6, "name": "FStab", "module": 5}, {"phase": 2, "name": "Format", "module": 5}, {"phase": 3, "name": "MountBoot", "module": 5}, {"phase": 3, "name": "MountRoot", "module": 5}, {"phase": 4, "name": "MountSpecials", "module": 5}, {"phase": 2, "name": "TuneVolumeFS", "module": 5}, {"phase": 8, "name": "UnmountRoot", "module": 5}, {"phase": 0, "name": "CheckHostDependencies", "module": 6}, {"phase": 0, "name": "HostDependencies", "module": 6}, {"phase": 6, "name": "AddExpandRoot", "module": 7}, {"phase": 6, "name": "AddSSHKeyGeneration", "module": 7}, {"phase": 6, "name": "AdjustExpandRootScript", "module": 7}, {"phase": 6, "name": "InstallInitScripts", "module": 7}, {"phase": 6, "name": "RemoveHWClock", "module": 7}, {"phase": 5, "name": "GenerateLocale", "module": 8}, {"phase": 0, "name": "LocaleBootstrapPackage", "module": 8}, {"phase": 6, "name": "SetTimezone", "module": 8}, {"phase": 1, "name": "Create", "module": 9}, {"phase": 9, "name": "MoveImage", "module": 9}, {"phase": 6, "name": "ConfigureNetworkIF", "module": 10}, {"phase": 6, "name": "RemoveDNSInfo", "module": 10}, {"phase": 6, "name": "RemoveHostname", "module": 10}, {"phase": 0, "name": "AddManifestPackages", "module": 11}, {"phase": 5, "name": "InstallPackages", "module": 11}, {"phase": 2, "name": "MapPartitions", "module": 12}, {"phase": 2, "name": "PartitionVolume", "module": 12}, {"phase": 8, "name": "UnmapPartitions", "module": 12}, {"phase": 6, "name": "DisableSSHDNSLookup", "module": 13}, {"phase": 6, "name": "DisableSSHPasswordAuthentication", "module": 13}, {"phase": 6, "name": "EnableShadowConfig", "module": 13}, {"phase": 1, "name": "Attach", "module": 14}, {"phase": 10, "name": "Delete", "module": 14}, {"phase": 8, "name": "Detach", "module": 14}, {"phase": 0, "name": "CreateWorkspace", "module": 15}, {"phase": 10, "name": "DeleteWorkspace", "module": 15}, {"phase": 0, "name": "AddSudoPackage", "module": 16}, {"phase": 6, "name": "AdminUserCredentials", "module": 16}, {"phase": 6, "name": "CreateAdminUser", "module": 16}, {"phase": 6, "name": "DisableRootLogin", "module": 16}, {"phase": 6, "name": "PasswordlessSudo", "module": 16}, {"phase": 10, "name": "WriteMetadata", "module": 17}, {"phase": 0, "name": "AddBackports", "module": 18}, {"phase": 0, "name": "AddCloudInitPackages", "module": 18}, {"phase": 6, "name": "DisableModules", "module": 18}, {"phase": 6, "name": "SetMetadataSource", "module": 18}, {"phase": 6, "name": "SetUsername", "module": 18}, {"phase": 6, "name": "ImageExecuteCommand", "module": 19}, {"phase": 4, "name": "AddFolderMounts", "module": 20}, {"phase": 0, "name": "CheckVMWareDMCommand", "module": 20}, {"phase": 0, "name": "CheckZerofreePath", "module": 20}, {"phase": 7, "name": "RemoveFolderMounts", "module": 20}, {"phase": 8, "name": "ShrinkVolume", "module": 20}, {"phase": 8, "name": "Zerofree", "module": 20}, {"phase": 0, "name": "AddONEContextPackage", "module": 21}, {"phase": 6, "name": "OpenNebulaContext", "module": 21}, {"phase": 5, "name": "CopyImage", "module": 22}, {"phase": 1, "name": "CreateFromImage", "module": 22}, {"phase": 1, "name": "CreateFromSnapshot", "module": 22}, {"phase": 5, "name": "Snapshot", "module": 22}, {"phase": 0, "name": "AddPackages", "module": 23}, {"phase": 6, "name": "ApplyPuppetManifest", "module": 23}, {"phase": 0, "name": "CheckPaths", "module": 23}, {"phase": 6, "name": "CopyPuppetAssets", "module": 23}, {"phase": 6, "name": "SetRootPassword", "module": 24}, {"phase": 0, "name": "AddUnattendedUpgradesPackage", "module": 25}, {"phase": 6, "name": "EnablePeriodicUpgrades", "module": 25}, {"phase": 6, "name": "AddInsecurePublicKey", "module": 26}, {"phase": 0, "name": "AddPackages", "module": 26}, {"phase": 0, "name": "CheckBoxPath", "module": 26}, {"phase": 0, "name": "CreateVagrantBoxDir", "module": 26}, {"phase": 6, "name": "CreateVagrantUser", "module": 26}, {"phase": 9, "name": "PackageBox", "module": 26}, {"phase": 6, "name": "PasswordlessSudo", "module": 26}, {"phase": 10, "name": "RemoveVagrantBoxDir", "module": 26}, {"phase": 6, "name": "SetHostname", "module": 26}, {"phase": 6, "name": "SetRootPassword", "module": 26}, {"phase": 0, "name": "AMIName", "module": 27}, {"phase": 9, "name": "BundleImage", "module": 27}, {"phase": 9, "name": "RegisterAMI", "module": 27}, {"phase": 10, "name": "RemoveBundle", "module": 27}, {"phase": 9, "name": "UploadImage", "module": 27}, {"phase": 6, "name": "ConfigurePVGrub", "module": 28}, {"phase": 0, "name": "Connect", "module": 29}, {"phase": 0, "name": "GetCredentials", "module": 29}, {"phase": 1, "name": "Attach", "module": 30}, {"phase": 1, "name": "Create", "module": 30}, {"phase": 9, "name": "Snapshot", "module": 30}, {"phase": 6, "name": "S3FStab", "module": 31}, {"phase": 0, "name": "GetInfo", "module": 32}, {"phase": 0, "name": "HostDependencies", "module": 32}, {"phase": 6, "name": "AddEC2InitScripts", "module": 33}, {"phase": 0, "name": "AddBuildEssentialPackage", "module": 34}, {"phase": 6, "name": "EnableDHCPCDDNS", "module": 34}, {"phase": 5, "name": "InstallEnhancedNetworking", "module": 34}, {"phase": 0, "name": "DefaultPackages", "module": 35}, {"phase": 0, "name": "DefaultPackages", "module": 36}, {"phase": 6, "name": "VirtIO", "module": 37}, {"phase": 5, "name": "AddGuestAdditionsPackages", "module": 38}, {"phase": 0, "name": "CheckGuestAdditionsPath", "module": 38}, {"phase": 5, "name": "InstallGuestAdditions", "module": 38}, {"phase": 0, "name": "DefaultPackages", "module": 39}], "links": [{"source": 32, "target": 58, "definer": 32}, {"source": 35, "target": 38, "definer": 35}, {"source": 36, "target": 38, "definer": 36}, {"source": 39, "target": 38, "definer": 39}, {"source": 43, "target": 56, "definer": 43}, {"source": 50, "target": 27, "definer": 50}, {"source": 52, "target": 58, "definer": 52}, {"source": 57, "target": 60, "definer": 57}, {"source": 76, "target": 2, "definer": 76}, {"source": 78, "target": 58, "definer": 78}, {"source": 82, "target": 56, "definer": 82}, {"source": 83, "target": 110, "definer": 83}, {"source": 86, "target": 47, "definer": 86}, {"source": 86, "target": 46, "definer": 86}, {"source": 99, "target": 60, "definer": 99}, {"source": 105, "target": 60, "definer": 105}, {"source": 115, "target": 33, "definer": 115}, {"source": 116, "target": 38, "definer": 116}, {"source": 123, "target": 49, "definer": 123}, {"source": 1, "target": 0, "definer": 0}, {"source": 40, "target": 3, "definer": 3}, {"source": 9, "target": 3, "definer": 3}, {"source": 3, "target": 4, "definer": 4}, {"source": 5, "target": 4, "definer": 4}, {"source": 7, "target": 9, "definer": 9}, {"source": 0, "target": 10, "definer": 10}, {"source": 0, "target": 11, "definer": 11}, {"source": 26, "target": 14, "definer": 14}, {"source": 26, "target": 15, "definer": 15}, {"source": 17, "target": 16, "definer": 16}, {"source": 0, "target": 22, "definer": 22}, {"source": 29, "target": 23, "definer": 23}, {"source": 32, "target": 25, "definer": 25}, {"source": 23, "target": 28, "definer": 28}, {"source": 24, "target": 29, "definer": 29}, {"source": 16, "target": 30, "definer": 30}, {"source": 27, "target": 31, "definer": 31}, {"source": 34, "target": 33, "definer": 33}, {"source": 38, "target": 37, "definer": 37}, {"source": 0, "target": 48, "definer": 48}, {"source": 4, "target": 49, "definer": 49}, {"source": 51, "target": 50, "definer": 50}, {"source": 32, "target": 52, "definer": 52}, {"source": 0, "target": 61, "definer": 61}, {"source": 38, "target": 62, "definer": 62}, {"source": 0, "target": 68, "definer": 68}, {"source": 67, "target": 68, "definer": 68}, {"source": 16, "target": 73, "definer": 73}, {"source": 58, "target": 77, "definer": 77}, {"source": 32, "target": 78, "definer": 78}, {"source": 52, "target": 78, "definer": 78}, {"source": 49, "target": 81, "definer": 81}, {"source": 125, "target": 81, "definer": 81}, {"source": 49, "target": 84, "definer": 84}, {"source": 125, "target": 84, "definer": 84}, {"source": 0, "target": 85, "definer": 85}, {"source": 88, "target": 86, "definer": 86}, {"source": 0, "target": 90, "definer": 90}, {"source": 96, "target": 92, "definer": 92}, {"source": 0, "target": 93, "definer": 93}, {"source": 59, "target": 95, "definer": 95}, {"source": 94, "target": 95, "definer": 95}, {"source": 108, "target": 102, "definer": 102}, {"source": 112, "target": 104, "definer": 104}, {"source": 106, "target": 104, "definer": 104}, {"source": 103, "target": 106, "definer": 106}, {"source": 109, "target": 108, "definer": 108}, {"source": 114, "target": 108, "definer": 108}, {"source": 111, "target": 110, "definer": 110}, {"source": 0, "target": 117, "definer": 117}, {"source": 0, "target": 120, "definer": 120}, {"source": 0, "target": 121, "definer": 121}, {"source": 49, "target": 125, "definer": 125}, {"source": 0, "target": 126, "definer": 126}]} \ No newline at end of file diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee new file mode 100644 index 0000000..d9d3b94 --- /dev/null +++ b/docs/_static/taskoverview.coffee @@ -0,0 +1,177 @@ +class window.TaskOverview + viewBoxHeight = 800 + viewBoxWidth = 200 + margins = + top: 100 + left: 50 + bottom: 100 + right: 50 + gravity = + lateral: .1 + longitudinal: .2 + + length = ([x,y]) -> Math.sqrt(x*x + y*y) + sum = ([x1,y1], [x2,y2]) -> [x1+x2, y1+y2] + diff = ([x1,y1], [x2,y2]) -> [x1-x2, y1-y2] + prod = ([x,y], scalar) -> [x*scalar, y*scalar] + div = ([x,y], scalar) -> [x/scalar, y/scalar] + unit = (vector) -> div(vector, length(vector)) + scale = (vector, scalar) -> prod(unit(vector), scalar) + position = (coord, vector) -> [coord, sum(coord, vector)] + + free = ([coord1, coord2]) -> diff(coord2, coord1) + pmult = (pvector=[coord1, _], scalar) -> position(coord1, prod(free(pvector), scalar)) + pdiv = (pvector=[coord1, _], scalar) -> position(coord1, div(free(pvector), scalar)) + + constructor: ({@selector}) -> + @svg = d3.select(@selector) + .attr('viewBox', "0 0 #{viewBoxWidth} #{viewBoxHeight}") + d3.json '_static/graph.json', @buildGraph + + buildGraph: (error, @data) => + @createDefinitions() + taskLayout = @createNodes() + taskLayout.start() + + createDefinitions: () -> + definitions = @svg.append 'defs' + arrow = definitions.append('marker') + arrow.attr('id', 'right-arrowhead') + .attr('refX', arrowHeight = 4) + .attr('refY', (arrowWidth = 6) / 2) + .attr('markerWidth', arrowHeight) + .attr('markerHeight', arrowWidth) + .attr('orient', 'auto') + .append('path').attr('d', "M0,0 V#{arrowWidth} L#{arrowHeight},#{arrowWidth/2} Z") + + partitionKey = 'phase' + nodeColorKey = 'module' + + keyMap: + phase: 'phases' + module: 'modules' + partition: (key, idx) -> + return @data[@keyMap[key]] + + nodeRadius = 10 + nodePadding = 10 + createNodes: () -> + options = + gravity: 0 + linkDistance: 50 + linkStrength: .8 + charge: -130 + size: [viewBoxWidth, viewBoxHeight] + + layout = d3.layout.force() + layout[option](value) for option, value of options + + array_sum = (list) -> list.reduce(((a,b) -> a + b), 0) + partitioning = + nonLinear: (groupCounts, range, k=2) -> + ratios = (Math.pow(count, 1/k) for count in groupCounts) + fraction = range / (array_sum ratios) + return (fraction * ratio for ratio in ratios) + linear: (groups, range) -> + fraction = range / groups + return (fraction for _ in [0..groups]) + offset: (ranges, i) -> (array_sum ranges.slice 0, i) + ranges[i] / 2 + + layout.nodes @data.nodes + layout.links @data.links + + grouping = d3.nest().key((d) -> d[partitionKey]).sortKeys(d3.ascending) + + widths = partitioning.nonLinear((d.values for d in grouping.rollup((d) -> d.length).entries(@data.nodes)), + viewBoxWidth - margins.left - margins.right) + heights = partitioning.nonLinear((d.values for d in grouping.rollup((d) -> d.length).entries(@data.nodes)), + viewBoxHeight - margins.top - margins.bottom, 4) + for node in @data.nodes + # node.cx = margins.left + partitioning.offset(widths, node[partitionKey]) + # node.cy = viewBoxHeight / 2 + margins.top + node.cx = viewBoxWidth / 2 + margins.left + node.cy = margins.top + partitioning.offset(heights, node[partitionKey]) + node.radius = nodeRadius + + + groups = d3.nest().key((d) -> d[partitionKey]) + .sortKeys(d3.ascending) + .entries(layout.nodes()) + + hullColors = d3.scale.category20() + nodeColors = d3.scale.category20c() + + hulls = @svg.append('g').attr('class', 'hulls') + .selectAll('path').data(groups).enter() + .append('path').attr('id', (d) -> "hull-#{d.key}") + .style + 'fill': (d, i) -> hullColors(i) + 'stroke': (d, i) -> hullColors(i) + + hullLabels = @svg.append('g').attr('class', 'hull-labels') + .selectAll('text').data(groups).enter() + .append('text') + hullLabels.append('textPath').attr('xlink:href', (d) -> "#hull-#{d.key}") + .text((d) => @partition(partitionKey)[d.key].name) + + links = @svg.append('g').attr('class', 'links') + .selectAll('line').data(layout.links()).enter() + .append('line').attr('marker-end', 'url(#right-arrowhead)') + + nodes = @svg.append('g').attr('class', 'nodes') + .selectAll('g.partition').data(groups).enter() + .append('g').attr('class', 'partition') + .selectAll('circle').data((d) -> d.values).enter() + .append('circle').attr('r', (d) -> d.radius) + .style('fill', (d, i) -> nodeColors(d[nodeColorKey])) + .call(layout.drag) + .on('mouseover', (d) -> (labels.filter (l) -> d is l).classed 'hover', true) + .on('mouseout', (d) -> (labels.filter (l) -> d is l).classed 'hover', false) + + labels = @svg.append('g').attr('class', 'node-labels') + .selectAll('g.partition').data(groups).enter() + .append('g').attr('class', 'partition') + .selectAll('text').data((d) -> d.values).enter() + .append('text').text((d) -> d.name) + .attr('transform', (d) -> offset=-(d.radius + 5); "translate(0,#{offset})") + + rotate = (x, n) -> + n = n % x.length + x.slice(0,-n).reverse().concat(x.slice(-n).reverse()).reverse() + circle_coords = (parts) -> + partSize = 2*Math.PI/parts + return (for i in [0..parts] + theta = partSize*i + [(Math.cos theta), (Math.sin theta)]) + hullPointMatrix = (prod(v, nodeRadius*2) for v in circle_coords(16)) + hullBoundaries = (d) -> + nodePoints = d.values.map (i) -> [i.x, i.y] + padded_points = [] + padded_points.push sum(p, v) for v in hullPointMatrix for p in nodePoints + points = d3.geom.hull(padded_points) + points = rotate points, Math.floor -points.length / 2.5 + "M#{points.join('L')}Z" + + gravity_fn = (alpha) => + (d) -> + d.x += (d.cx - d.x) * alpha * gravity.lateral + d.y += (d.cy - d.y) * alpha * gravity.longitudinal + + layout.on 'tick', (e) => + hulls.attr('d', hullBoundaries) + nodes.each gravity_fn(e.alpha) + nodes.attr + cx: ({x}) -> x + cy: ({y}) -> y + labels.each gravity_fn(e.alpha) + labels.attr + x: ({x}) -> x + y: ({y}) -> y + links.each ({source:{x:x1,y:y1},target:{x:x2,y:y2}}, i) -> + [x,y] = scale(free([[x1,y1], [x2,y2]]), nodeRadius) + @setAttribute 'x1', x1 + x + @setAttribute 'y1', y1 + y + @setAttribute 'x2', x2 - x + @setAttribute 'y2', y2 - y + + return layout diff --git a/docs/_static/taskoverview.less b/docs/_static/taskoverview.less new file mode 100644 index 0000000..2903816 --- /dev/null +++ b/docs/_static/taskoverview.less @@ -0,0 +1,28 @@ +#taskoverview-graph { + g.hulls path { + opacity: 0.25; + } + g.hull-labels text { + } + g.nodes circle { + stroke: #000000; + opacity: .9; + stroke-width: 1.5px; + } + g.node-labels text { + pointer-events: none; + font: 10px sans-serif; + text-anchor: middle; + text-shadow: 0 0 2px #FFFFFF; + font-weight: bold; + opacity: 0; + &.hover { + transition: opacity .5s; + opacity: .9; + } + } + g.links line { + stroke: #999; + stroke-opacity: .6; + } +} diff --git a/docs/guidelines.rst b/docs/guidelines.rst index cd798c9..f17c330 100644 --- a/docs/guidelines.rst +++ b/docs/guidelines.rst @@ -1,3 +1,4 @@ + Development guidelines ====================== The following guidelines should serve as general advice when diff --git a/docs/index.rst b/docs/index.rst index 9e928b0..c706b12 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ Contents: plugins/index providers/index guidelines + taskoverview Indices and tables ================== diff --git a/docs/taskoverview.rst b/docs/taskoverview.rst new file mode 100644 index 0000000..e84680e --- /dev/null +++ b/docs/taskoverview.rst @@ -0,0 +1,15 @@ + +Taskoverview +============ + +.. raw:: html + + + + + + + +