Add support for installing to primary partitions, NTFS, partclone

- refactored parititioning code to support installing to primary partitions
- adds "requires_partition_number" property to partitions.json for partitions
  that need to be at a fixed partition number (e.g. need to be at primary
  partition 3)
- adds "offset" property to partitions.json for partitions that need to be
  at a fixed offset.
- now does rewrite the MBR
- outsources the writing of the partition table to sfdisk
- enables NTFS support (through fuse/ntfs-3g)
- adds OsInfo PartitionInfo model classes that parse the json files once and
  keep the information in normal variables
- tries to reduce wasted space due to gaps between partitions
- adds support for keeping partition unformatted (filesystem_type: "unformatted")
- adds support for specifying hexadecimal partition type number (partition_type: "07")
- adds support for marking partitions bootable (active: true)
- kernel config: enable FUSE and UDF
- zero out first sector of each partition (to prevent old file system label being detected)
- parted: surpress warning about its file system manipulation code not being robust that
  distracts from real errors
- adds support for partclone disk partition images
- adds support for patching Windows BCD files

Author: maxnet

buildroot/.config

@@ -1,6 +1,6 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Buildroot 2015.02-git-g661b7ed-dirty Configuration
+# Buildroot 2015.02-git-ge7d8e80-dirty Configuration
 #
 BR2_HAVE_DOT_CONFIG=y
 
@@ -337,6 +337,7 @@ BR2_PACKAGE_BUSYBOX_CONFIG="package/busybox/busybox.config"
 BR2_PACKAGE_BUSYBOX_SHOW_OTHERS=y
 # BR2_PACKAGE_BUSYBOX_WATCHDOG is not set
 BR2_PACKAGE_ARORA=y
+BR2_PACKAGE_PARTCLONE=y
 BR2_PACKAGE_RECOVERY=y
 
 #
@@ -535,7 +536,9 @@ BR2_PACKAGE_E2FSPROGS_RESIZE2FS=y
 # BR2_PACKAGE_MTD is not set
 BR2_PACKAGE_MTOOLS=y
 # BR2_PACKAGE_NFS_UTILS is not set
-# BR2_PACKAGE_NTFS_3G is not set
+BR2_PACKAGE_NTFS_3G=y
+# BR2_PACKAGE_NTFS_3G_ENCRYPTED is not set
+BR2_PACKAGE_NTFS_3G_NTFSPROGS=y
 # BR2_PACKAGE_SIMICSFS is not set
 # BR2_PACKAGE_SQUASHFS is not set
 # BR2_PACKAGE_SSHFS is not set

buildroot/kernelconfig-recovery.armv6

@@ -113,6 +113,8 @@ CONFIG_DMA_BCM2708=y
 # CONFIG_IOMMU_SUPPORT is not set
 CONFIG_EXT4_FS=y
 CONFIG_FANOTIFY=y
+CONFIG_FUSE_FS=y
+CONFIG_UDF_FS=y
 CONFIG_FSCACHE=y
 CONFIG_CACHEFILES=y
 CONFIG_MSDOS_FS=y

buildroot/kernelconfig-recovery.armv7

@@ -118,6 +118,8 @@ CONFIG_DMA_BCM2708=y
 # CONFIG_IOMMU_SUPPORT is not set
 CONFIG_EXT4_FS=y
 CONFIG_FANOTIFY=y
+CONFIG_FUSE_FS=y
+CONFIG_UDF_FS=y
 CONFIG_FSCACHE=y
 CONFIG_CACHEFILES=y
 CONFIG_MSDOS_FS=y

buildroot/package/Config.in

@@ -2,6 +2,7 @@ menu "Target packages"
 
 	source "package/busybox/Config.in"
 	source "package/arora/Config.in"
+	source "package/partclone/Config.in"
 	source "package/recovery/Config.in"
 
 menu "Audio and video applications"

buildroot/package/partclone/01-remove-mcheck.patch

@@ -0,0 +1,11 @@
+--- partclone-0.2.84.orig/src/main.c    2015-10-13 08:27:56.000000000 +0200
++++ partclone-0.2.84/src/main.c 2015-10-27 17:09:32.324546758 +0100
+@@ -20,7 +20,6 @@
+ #include <sys/types.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+-#include <mcheck.h>
+ #include <stdint.h>
+ #include <stdarg.h>
+ #include <string.h>
+

buildroot/package/partclone/Config.in

@@ -0,0 +1,10 @@
+config BR2_PACKAGE_PARTCLONE
+	bool "partclone"
+	help
+	  Partclone
+	  Note: only installs partclone.restore
+
+	  http://partclone.org
+
+comment "partclone requires a toolchain with LARGEFILE+WCHAR support"
+        depends on !(BR2_LARGEFILE && BR2_USE_WCHAR)

buildroot/package/partclone/partclone.mk

@@ -0,0 +1,21 @@
+################################################################################
+#
+# partclone
+#
+################################################################################
+
+PARTCLONE_VERSION = 0.2.84
+PARTCLONE_SOURCE = partclone-$(PARTCLONE_VERSION).tar.gz
+PARTCLONE_SITE = http://sourceforge.net/projects/partclone/files/testing/src
+PARTCLONE_LICENSE = GPLv2
+PARTCLONE_LICENSE_FILES = COPYING
+
+define PARTCLONE_BUILD_CMDS
+	$(MAKE) CC="$(TARGET_CC)" LD="$(TARGET_LD)" -C $(@D)/src partclone.restore 
+endef
+
+define PARTCLONE_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 $(@D)/src/partclone.restore $(TARGET_DIR)/usr/bin/partclone.restore
+endef
+
+$(eval $(autotools-package))

recovery/config.h

@@ -33,13 +33,20 @@
  * If files they are larger than number of MB, try resizing the FAT partition instead */
 #define MAXIMUM_BOOTFILES_SIZE  64
 
-#define SETTINGS_PARTITION  "/dev/mmcblk0p3"
-#define SETTINGS_PARTITION_SIZE  (32 * 2048)
+/* Partitioning settings */
+#define PARTITION_ALIGNMENT  8192
+#define PARTITION_GAP  2
+/* Allow partitions to be shrinked PARTITION_GAP sectors
+   if that prevents having a 4 MiB gap between the next one */
+#define SHRINK_PARTITIONS_TO_MINIMIZE_GAPS
+
+#define SETTINGS_PARTITION  "/dev/mmcblk0p5"
+#define SETTINGS_PARTITION_SIZE  (32 * 2048 - PARTITION_GAP)
 
 /* If the image name matches this exactly, mark it as recommended */
 #define RECOMMENDED_IMAGE "Raspbian"
 
-#define FAT_PARTITION_OF_IMAGE  "/dev/mmcblk0p5"
+#define FAT_PARTITION_OF_IMAGE  "/dev/mmcblk0p6"
 
 /* RiscOS magic */
 #define RISCOS_OFFSET_KEY "riscos_offset"

recovery/initdrivethread.cpp

@@ -10,6 +10,7 @@
 #include "initdrivethread.h"
 #include "mbr.h"
 #include "util.h"
+#include "config.h"
 #include <QProcess>
 #include <QFile>
 #include <QDir>
@@ -37,30 +38,22 @@ void InitDriveThread::run()
     emit statusUpdate(tr("Mounting FAT partition"));
     mountSystemPartition();
 
-    //if (sizeofBootFilesInKB() > (MAXIMUM_BOOTFILES_SIZE*1024))
-    //{
-        //emit error(tr("SD card contains extra files that do not belong to this distribution. Please copy them to another disk and delete them from card."));
-        //return;
-
-        // Try to resize existing partitions
-        if (!method_resizePartitions())
-        {
-            return;
-        }
-//}
-//  else
-//  {
-        //      // Reformat the drive
-        //if (!method_reformatDrive())
-            //{
-            //return;
-            //      }
-// }
+    // Try to resize existing partitions
+    if (!method_resizePartitions())
+        return;
 
     emit statusUpdate(tr("Formatting settings partition"));
     if (!formatSettingsPartition())
     {
         emit error(tr("Error formatting settings partition"));
+        return;
+    }
+
+    emit statusUpdate(tr("Mounting FAT partition"));
+    if (!mountSystemPartition())
+    {
+        emit error(tr("Error mounting system partition."));
+        return;
     }
 
     dir.mkdir("/mnt/os");
@@ -128,53 +121,10 @@ void InitDriveThread::run()
     emit completed();
 }
 
-bool InitDriveThread::method_reformatDrive()
-{
-    emit statusUpdate(tr("Saving boot files to memory"));
-    if (!saveBootFiles() )
-    {
-        emit error(tr("Error saving boot files to memory. SD card may be damaged."));
-        return false;
-    }
-    if (!umountSystemPartition())
-    {
-        emit error(tr("Error unmounting system partition."));
-        return false;
-    }
-
-    emit statusUpdate(tr("Zeroing partition table"));
-    if (!zeroMbr())
-    {
-        emit error(tr("Error zero'ing MBR/GPT. SD card may be broken or advertising wrong capacity."));
-        return false;
-    }
-
-    emit statusUpdate(tr("Creating partitions"));
-
-    if (!partitionDrive())
-    {
-        emit error(tr("Error partitioning"));
-        return false;
-    }
-
-    emit statusUpdate(tr("Formatting boot partition (fat)"));
-    if (!formatBootPartition())
-    {
-       emit error(tr("Error formatting boot partition (fat)"));
-       return false;
-    }
-
-    emit statusUpdate(tr("Copying boot files to storage"));
-    mountSystemPartition();
-    restoreBootFiles();
-
-    return true;
-}
-
 bool InitDriveThread::method_resizePartitions()
 {
     int newStartOfRescuePartition = getFileContents("/sys/class/block/mmcblk0p1/start").trimmed().toInt();
-    int newSizeOfRescuePartition  = sizeofBootFilesInKB()/1000 + 100;
+    int newSizeOfRescuePartition  = sizeofBootFilesInKB()*1.024/1000 + 100;
 
     if (!umountSystemPartition())
     {
@@ -250,12 +200,17 @@ bool InitDriveThread::method_resizePartitions()
      * only move it when it is not aligned on a MiB boundary already */
     if (newStartOfRescuePartition < 2048 || newStartOfRescuePartition % 2048 != 0)
     {
-        newStartOfRescuePartition = 8192; /* 4 MiB */
+        newStartOfRescuePartition = PARTITION_ALIGNMENT; /* 4 MiB */
     }
 
     QString cmd = "/usr/sbin/parted --script /dev/mmcblk0 resize 1 "+QString::number(newStartOfRescuePartition)+"s "+QString::number(newSizeOfRescuePartition)+"M";
     qDebug() << "Executing" << cmd;
     QProcess p;
+    QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
+    /* Suppress parted's big fat warning about its file system manipulation code not being robust.
+       It distracts from any real error messages that may follow it. */
+    env.insert("PARTED_SUPPRESS_FILE_SYSTEM_MANIPULATION_WARNING", "1");
+    p.setProcessEnvironment(env);
     p.setProcessChannelMode(p.MergedChannels);
     p.start(cmd);
     p.closeWriteChannel();
@@ -271,33 +226,25 @@ bool InitDriveThread::method_resizePartitions()
 
     emit statusUpdate(tr("Creating extended partition"));
 
-    mbr_table extended_mbr;
     QByteArray partitionTable;
     int startOfOurPartition = getFileContents("/sys/class/block/mmcblk0p1/start").trimmed().toInt();
     int sizeOfOurPartition  = getFileContents("/sys/class/block/mmcblk0p1/size").trimmed().toInt();
     int startOfExtended = startOfOurPartition+sizeOfOurPartition;
-    // Align on 4 MiB boundary
-    startOfExtended += 8192-(startOfExtended % 8192);
 
-    int sizeOfDisk = getFileContents("/sys/class/block/mmcblk0/size").trimmed().toULongLong();
-    int sizeOfExtended = sizeOfDisk - startOfExtended - SETTINGS_PARTITION_SIZE ;
-    int startOfSettings = startOfExtended+sizeOfExtended;
+    // Align start of settings partition on 4 MiB boundary
+    int startOfSettings = startOfExtended + PARTITION_GAP;
+    if (startOfSettings % PARTITION_ALIGNMENT != 0)
+         startOfSettings += PARTITION_ALIGNMENT-(startOfSettings % PARTITION_ALIGNMENT);
 
+    // Primary partitions
     partitionTable  = QByteArray::number(startOfOurPartition)+","+QByteArray::number(sizeOfOurPartition)+",0E\n"; /* FAT partition */
-    partitionTable += QByteArray::number(startOfExtended)+","+QByteArray::number(sizeOfExtended)+",X\n"; /* Extended partition with all remaining space */
-    partitionTable += QByteArray::number(startOfSettings)+",,L\n"; /* Settings partition */
+    partitionTable += QByteArray::number(startOfExtended)+",,X\n"; /* Extended partition with all remaining space */
     partitionTable += "0,0\n";
+    partitionTable += "0,0\n";
+    // Logical partitions
+    partitionTable += QByteArray::number(startOfSettings)+","+QByteArray::number(SETTINGS_PARTITION_SIZE)+",L\n"; /* Settings partition */
     qDebug() << "Writing partition table" << partitionTable;
 
-    /* Write out empty extended partition table with signature */
-    memset(&extended_mbr, 0, sizeof extended_mbr);
-    extended_mbr.signature[0] = 0x55;
-    extended_mbr.signature[1] = 0xAA;
-    f.open(f.ReadWrite);
-    f.seek(startOfExtended*512);
-    f.write((char *) &extended_mbr, sizeof(extended_mbr));
-    f.close();
-
     /* Let sfdisk write a proper partition table */
     cmd = QString("/sbin/sfdisk -uS /dev/mmcblk0");
     QProcess proc;
@@ -325,28 +272,9 @@ bool InitDriveThread::method_resizePartitions()
 
     QProcess::execute("/sbin/mlabel p:RECOVERY");
 
-    emit statusUpdate(tr("Mounting FAT partition"));
-    if (!mountSystemPartition())
-    {
-        emit error(tr("Error mounting system partition."));
-        return false;
-    }
-
     return true;
 }
 
-bool InitDriveThread::saveBootFiles()
-{
-    return QProcess::execute("cp -a /mnt /tmp") == 0;
-}
-
-bool InitDriveThread::restoreBootFiles()
-{
-    bool status = QProcess::execute("cp -a /tmp/mnt /") == 0;
-    QProcess::execute("rm -rf /tmp/mnt");
-    return status;
-}
-
 int InitDriveThread::sizeofBootFilesInKB()
 {
     QProcess proc;
@@ -397,52 +325,6 @@ bool InitDriveThread::zeroMbr()
         && QProcess::execute("/bin/dd conv=fsync count=8 bs=512 if=/dev/zero seek="+QString::number(sizeofSDCardInBlocks()-8)+" of=/dev/mmcblk0") == 0;
 }
 
-bool InitDriveThread::partitionDrive()
-{
-    /* Partition layout:
-     *
-     * First 1MB (2048 blocks) kept empty for alignment
-     * Followed by FAT partition of RESCUE_PARTITION_SIZE (default 1 GB)
-     * Followed by extended partition spanning remainder of space
-     * Followed by NOOBS persistent settings partition
-     */
-    QByteArray partitionTable;
-    int rescueBlocks = RESCUE_PARTITION_SIZE*1024*2;
-
-    mbr_table extended_mbr;
-    int startOfExtended = 2048+rescueBlocks;
-    int sizeOfDisk = getFileContents("/sys/class/block/mmcblk0/size").trimmed().toULongLong();
-    int sizeOfExtended = sizeOfDisk - startOfExtended - SETTINGS_PARTITION_SIZE;
-    int startOfSettings = startOfExtended + sizeOfExtended;
-
-    partitionTable = "2048,"+QByteArray::number(rescueBlocks)+",0E\n"; /* FAT partition */
-    partitionTable += QByteArray::number(startOfExtended)+","+QByteArray::number(sizeOfExtended)+",X\n"; /* Extended partition with all remaining space */
-    partitionTable += QByteArray::number(startOfSettings)+",,L\n"; /* Settings partition */
-    partitionTable += "0,0\n";
-
-    /* Write out empty extended partition table with signature */
-    memset(&extended_mbr, 0, sizeof extended_mbr);
-    extended_mbr.signature[0] = 0x55;
-    extended_mbr.signature[1] = 0xAA;
-    QFile f("/dev/mmcblk0");
-    f.open(f.ReadWrite);
-    f.seek(startOfExtended*512);
-    f.write((char *) &extended_mbr, sizeof(extended_mbr));
-    f.close();
-
-    /* Write main partition table */
-    QString cmd = QString("/sbin/sfdisk -uS /dev/mmcblk0");
-    QProcess proc;
-    proc.setProcessChannelMode(proc.MergedChannels);
-    proc.start(cmd);
-    proc.write(partitionTable);
-    proc.closeWriteChannel();
-    proc.waitForFinished(-1);
-    QThread::msleep(500);
-
-    return proc.exitCode() == 0;
-}
-
 #ifdef RISCOS_BLOB_FILENAME
 bool InitDriveThread::writeRiscOSblob()
 {

recovery/initdrivethread.h

@@ -23,17 +23,13 @@ class InitDriveThread : public QThread
 protected:
     virtual void run();
 
-    bool method_reformatDrive();
     bool method_resizePartitions();
-    bool saveBootFiles();
-    bool restoreBootFiles();
     int sizeofBootFilesInKB();
     int sizeofSDCardInBlocks();
     bool mountSystemPartition();
     bool umountSystemPartition();
     bool zeroMbr();
     bool formatBootPartition();
-    bool partitionDrive();
     bool formatSettingsPartition();
 #ifdef RISCOS_BLOB_FILENAME
     bool writeRiscOSblob();