Added resumable download

Author: saniales

cmd/arduino_lib.go

@@ -159,7 +159,7 @@ func executeDownloadCommand(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	libs, failed := purgeInvalidLibraries(parseLibArgs(args), status.(*libraries.StatusContext))
+	libs, failed := purgeInvalidLibraries(parseLibArgs(args), status.(libraries.StatusContext))
 	libraryResults := parallelLibDownloads(libs, true, "Downloaded")
 
 	for libFail, err := range failed {
@@ -226,7 +226,7 @@ func parallelLibDownloads(items map[*libraries.Library]string, forced bool, OkSt
 
 	for library, version := range items {
 		release := library.GetVersion(version)
-		if release != nil && !library.IsCached(version) || forced {
+		if forced || release != nil && !library.IsCached(version) || release.CheckLocalArchive() != nil {
 			var pBar *pb.ProgressBar
 			if textMode {
 				pBar = pb.StartNew(release.Size).SetUnits(pb.U_BYTES).Prefix(fmt.Sprintf("%-20s", library.Name))

common/checksums/checksums.go

@@ -0,0 +1,52 @@
+package checksums
+
+import (
+	"bytes"
+	"crypto"
+	"encoding/hex"
+	"hash"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/bcmi-labs/arduino-cli/common"
+)
+
+func getHashAlgoAndComponent(checksum string) (hash.Hash, []byte) {
+	components := strings.SplitN(checksum, ":", 2)
+	hashAlgo := components[0]
+	hashMid, err := hex.DecodeString(components[1])
+	if err != nil {
+		return nil, nil
+	}
+
+	hash := []byte(hashMid)
+	switch hashAlgo {
+	case "SHA-256":
+		return crypto.SHA256.New(), hash
+	case "SHA1":
+		return crypto.SHA1.New(), hash
+	case "MD5":
+		return crypto.MD5.New(), hash
+	default:
+		return nil, nil
+	}
+}
+
+// Match checks the checksum of a Release archive, in compliance with
+// What Checksum is expected.
+func Match(r common.Release) bool {
+	hash, content := getHashAlgoAndComponent(r.ExpectedChecksum())
+	filePath, err := r.ArchivePath()
+	if err != nil {
+		return false
+	}
+
+	file, err := os.Open(filePath)
+	if err != nil {
+		return false
+	}
+	defer file.Close()
+	io.Copy(hash, file)
+	return bytes.Compare(hash.Sum(nil), content) == 0
+}

common/functions.go

@@ -31,8 +31,9 @@ package common
 
 import (
 	"archive/zip"
-	"bytes"
+	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -102,7 +103,7 @@ func GetFolder(folder string, messageName string) (string, error) {
 }
 
 // Unzip extracts a zip file to a specified destination path.
-func Unzip(archive *zip.Reader, destination string) error {
+func Unzip(archive *zip.ReadCloser, destination string) error {
 	for _, file := range archive.File {
 		path := filepath.Join(destination, file.Name)
 		if file.FileInfo().IsDir() {
@@ -155,32 +156,44 @@ func TruncateDir(dir string) error {
 }
 
 // DownloadPackage downloads a package from arduino repository, applying a label for the progress bar.
-func DownloadPackage(URL string, downloadLabel string, progressBar *pb.ProgressBar, initialData []byte) ([]byte, error) {
+func DownloadPackage(URL string, downloadLabel string, progressBar *pb.ProgressBar, initialData *os.File, totalSize int) error {
 	client := http.DefaultClient
 
+	if initialData == nil {
+		return errors.New("Cannot fill a nil file pointer")
+	}
+
 	request, err := http.NewRequest("GET", URL, nil)
 	if err != nil {
-		return nil, fmt.Errorf("Cannot create HTTP request: %s", err)
+		return fmt.Errorf("Cannot create HTTP request: %s", err)
 	}
 
 	var initialSize int
-	if initialData == nil {
+	stats, err := initialData.Stat()
+	if err != nil {
 		initialSize = 0
 	} else {
-		initialSize = len(initialData)
+		fileSize := int(stats.Size())
+		if fileSize >= totalSize {
+			initialSize = 0
+		} else {
+			initialSize = fileSize
+		}
 	}
 
 	if initialSize > 0 {
 		request.Header.Add("Range", fmt.Sprintf("bytes=%d-", initialSize))
 	}
-	//TODO : how to add progressbar with resume download?
+
 	response, err := client.Do(request)
 
 	if err != nil {
-		return nil, fmt.Errorf("Cannot fetch %s. Response creation error", downloadLabel)
-	} else if response.StatusCode != 200 {
+		return fmt.Errorf("Cannot fetch %s. Response creation error", downloadLabel)
+	} else if response.StatusCode != 200 &&
+		response.StatusCode != 206 &&
+		response.StatusCode != 416 {
 		response.Body.Close()
-		return nil, fmt.Errorf("Cannot fetch %s. Source responded with a status %d code", downloadLabel, response.StatusCode)
+		return fmt.Errorf("Cannot fetch %s. Source responded with a status %d code", downloadLabel, response.StatusCode)
 	}
 	defer response.Body.Close()
 
@@ -190,16 +203,9 @@ func DownloadPackage(URL string, downloadLabel string, progressBar *pb.ProgressB
 		source = progressBar.NewProxyReader(response.Body)
 	}
 
-	body, err := ioutil.ReadAll(source)
+	_, err = io.Copy(initialData, source)
 	if err != nil {
-		return nil, fmt.Errorf("Cannot read response body")
-	}
-	var total []byte
-	if initialData != nil {
-		total = bytes.Join([][]byte{initialData, body}, nil)
-	} else {
-		total = body
+		return fmt.Errorf("Cannot read response body %s", err)
 	}
-
-	return total, nil
+	return nil
 }

common/interfaces.go

@@ -39,3 +39,9 @@ type StatusContext interface {
 	Names() []string               // Names Returns an array with all the names of the items.
 	Items() map[string]interface{} // Items Returns a map of all items with their names.
 }
+
+// Release represents a generic release.
+type Release interface {
+	ArchivePath() (string, error) // ArchivePath returns the fullPath of the Archive of this release.
+	ExpectedChecksum() string     // Checksum returns the expected checksum for this release.
+}

libraries/download.go

@@ -51,24 +51,15 @@ const (
 func DownloadAndCache(library *Library, progBar *pb.ProgressBar, version string) task.Wrapper {
 	return task.Wrapper{
 		Task: func() task.Result {
-			zipContent, err := downloadRelease(library, progBar, version)
+			err := downloadRelease(library, progBar, version)
 			if err != nil {
 				return task.Result{
 					Result: nil,
 					Error:  err,
 				}
 			}
-
-			zipArchive, err := prepareInstall(library, zipContent, version)
-			if err != nil {
-				return task.Result{
-					Result: nil,
-					Error:  err,
-				}
-			}
-
 			return task.Result{
-				Result: zipArchive,
+				Result: nil,
 				Error:  nil,
 			}
 		},
@@ -77,16 +68,20 @@ func DownloadAndCache(library *Library, progBar *pb.ProgressBar, version string)
 
 // DownloadLatest downloads Latest version of a library.
 func downloadLatest(library *Library, progBar *pb.ProgressBar) ([]byte, error) {
-	return downloadRelease(library, progBar, library.latestVersion())
+	return nil, downloadRelease(library, progBar, library.latestVersion())
 }
 
-func downloadRelease(library *Library, progBar *pb.ProgressBar, version string) ([]byte, error) {
+func downloadRelease(library *Library, progBar *pb.ProgressBar, version string) error {
 	release := library.GetVersion(version)
 	if release == nil {
-		return nil, errors.New("Invalid version number")
+		return errors.New("Invalid version number")
+	}
+	initialData, err := release.OpenLocalArchiveForDownload()
+	if err != nil {
+		return fmt.Errorf("Cannot get Archive file of this release : %s", err)
 	}
-	initialData := release.ReadLocalArchive()
-	return common.DownloadPackage(release.URL, fmt.Sprintf("library %s", library.Name), progBar, initialData)
+	defer initialData.Close()
+	return common.DownloadPackage(release.URL, fmt.Sprintf("library %s", library.Name), progBar, initialData, release.Size)
 }
 
 // DownloadLibrariesFile downloads the lib file from arduino repository.

libraries/install_uninstall.go

@@ -31,10 +31,8 @@ package libraries
 
 import (
 	"archive/zip"
-	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -95,15 +93,12 @@ func InstallLib(library *Library, version string) error {
 	}
 
 	cacheFilePath := filepath.Join(stagingFolder, release.ArchiveFileName)
-	content, err := ioutil.ReadFile(cacheFilePath)
-	if err != nil {
-		return err
-	}
 
-	zipArchive, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
+	zipArchive, err := zip.OpenReader(cacheFilePath)
 	if err != nil {
 		return err
 	}
+	defer zipArchive.Close()
 
 	err = common.Unzip(zipArchive, libFolder)
 	if err != nil {
@@ -124,29 +119,3 @@ func removeRelease(l *Library, r *Release) error {
 	path := filepath.Join(libFolder, fmt.Sprintf("%s-%s", name, r.Version))
 	return os.RemoveAll(path)
 }
-
-// prepareInstall move a downloaded library to a cache folder, before installation.
-func prepareInstall(library *Library, body []byte, version string) (*zip.Reader, error) {
-	reader := bytes.NewReader(body)
-	release := library.GetVersion(version)
-	if release == nil {
-		return nil, errors.New("Invalid version number")
-	}
-
-	archive, err := zip.NewReader(reader, int64(reader.Len()))
-	if err != nil {
-		return nil, fmt.Errorf("Cannot read downloaded archive")
-	}
-
-	// if I can read the archive I save it to staging folder.
-	stagingFolder, err := getDownloadCacheFolder()
-	if err != nil {
-		return nil, fmt.Errorf("Cannot get download cache folder")
-	}
-
-	err = ioutil.WriteFile(filepath.Join(stagingFolder, release.ArchiveFileName), body, 0666)
-	if err != nil {
-		return nil, fmt.Errorf("Cannot write download to cache folder, %s", err.Error())
-	}
-	return archive, nil
-}

libraries/libraries.go

@@ -31,6 +31,7 @@ package libraries
 
 import (
 	"bufio"
+	"errors"
 	"strconv"
 
 	"strings"
@@ -44,6 +45,7 @@ import (
 	"fmt"
 
 	"github.com/bcmi-labs/arduino-cli/common"
+	"github.com/bcmi-labs/arduino-cli/common/checksums"
 	"github.com/blang/semver"
 )
 
@@ -128,21 +130,59 @@ type Release struct {
 	Checksum        string `json:"checksum"`
 }
 
-// ReadLocalArchive reads the data from the local archive if present,
+// OpenLocalArchiveForDownload reads the data from the local archive if present,
 // and returns the []byte of the file content. Used by resume Download.
-func (r Release) ReadLocalArchive() []byte {
+// Creates an empty file if not found.
+func (r Release) OpenLocalArchiveForDownload() (*os.File, error) {
+	path, err := r.ArchivePath()
+	if err != nil {
+		return nil, err
+	}
+	stats, err := os.Stat(path)
+	if os.IsNotExist(err) || err == nil && int(stats.Size()) >= r.Size {
+		return os.Create(path)
+	}
+	return os.OpenFile(path, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
+}
+
+// ArchivePath returns the fullPath of the Archive of this release.
+func (r Release) ArchivePath() (string, error) {
 	staging, err := getDownloadCacheFolder()
 	if err != nil {
-		return nil
+		return "", err
+	}
+	return filepath.Join(staging, r.ArchiveFileName), nil
+}
+
+// CheckLocalArchive check for integrity of the local archive.
+func (r Release) CheckLocalArchive() error {
+	archivePath, err := r.ArchivePath()
+	if err != nil {
+		return err
 	}
-	path := filepath.Join(staging, r.ArchiveFileName)
-	content, err := ioutil.ReadFile(path)
-	// if the size is the same the archive has been downloaded and if we force redownload
-	// it won't work.
-	if err != nil || len(content) == r.Size {
-		return nil
+	stats, err := os.Stat(archivePath)
+	if os.IsNotExist(err) {
+		return errors.New("Archive does not exist")
 	}
-	return content
+	if err != nil {
+		return err
+	}
+	if int(stats.Size()) > r.Size {
+		return errors.New("Archive size does not match with specification of this release, assuming corruption")
+	}
+	if !r.checksumMatches() {
+		return errors.New("Checksum does not match, assuming corruption")
+	}
+	return nil
+}
+
+func (r Release) checksumMatches() bool {
+	return checksums.Match(r)
+}
+
+// ExpectedChecksum returns the expected checksum for this release.
+func (r Release) ExpectedChecksum() string {
+	return r.Checksum
 }
 
 /*