Windows: Updates Windows Vendoring

Updates windows dependent libraries for vendoing.

vendor/github.com/Microsoft/hcsshim/internal/lcow/constants.go

@@ -0,0 +1,9 @@
+package lcow
+
+const (
+	// DefaultScratchSizeGB is the size of the default LCOW scratch disk in GB
+	DefaultScratchSizeGB = 20
+
+	// defaultVhdxBlockSizeMB is the block-size for the scratch VHDx's this package can create.
+	defaultVhdxBlockSizeMB = 1
+)

vendor/github.com/Microsoft/hcsshim/internal/lcow/debug.go

@@ -0,0 +1,55 @@
+package lcow
+
+//func debugCommand(s string) string {
+//	return fmt.Sprintf(`echo -e 'DEBUG COMMAND: %s\\n--------------\\n';%s;echo -e '\\n\\n';`, s, s)
+//}
+
+// DebugLCOWGCS extracts logs from the GCS in LCOW. It's a useful hack for debugging,
+// but not necessarily optimal, but all that is available to us in RS3.
+//func (container *container) DebugLCOWGCS() {
+//	if logrus.GetLevel() < logrus.DebugLevel || len(os.Getenv("HCSSHIM_LCOW_DEBUG_ENABLE")) == 0 {
+//		return
+//	}
+
+//	var out bytes.Buffer
+//	cmd := os.Getenv("HCSSHIM_LCOW_DEBUG_COMMAND")
+//	if cmd == "" {
+//		cmd = `sh -c "`
+//		cmd += debugCommand("kill -10 `pidof gcs`") // SIGUSR1 for stackdump
+//		cmd += debugCommand("ls -l /tmp")
+//		cmd += debugCommand("cat /tmp/gcs.log")
+//		cmd += debugCommand("cat /tmp/gcs/gcs-stacks*")
+//		cmd += debugCommand("cat /tmp/gcs/paniclog*")
+//		cmd += debugCommand("ls -l /tmp/gcs")
+//		cmd += debugCommand("ls -l /tmp/gcs/*")
+//		cmd += debugCommand("cat /tmp/gcs/*/config.json")
+//		cmd += debugCommand("ls -lR /var/run/gcsrunc")
+//		cmd += debugCommand("cat /tmp/gcs/global-runc.log")
+//		cmd += debugCommand("cat /tmp/gcs/*/runc.log")
+//		cmd += debugCommand("ps -ef")
+//		cmd += `"`
+//	}
+
+//	proc, _, err := container.CreateProcessEx(
+//		&CreateProcessEx{
+//			OCISpecification: &specs.Spec{
+//				Process: &specs.Process{Args: []string{cmd}},
+//				Linux:   &specs.Linux{},
+//			},
+//			CreateInUtilityVm: true,
+//			Stdout:            &out,
+//		})
+//	defer func() {
+//		if proc != nil {
+//			proc.Kill()
+//			proc.Close()
+//		}
+//	}()
+//	if err != nil {
+//		logrus.Debugln("benign failure getting gcs logs: ", err)
+//	}
+//	if proc != nil {
+//		proc.WaitTimeout(time.Duration(int(time.Second) * 30))
+//	}
+//	logrus.Debugf("GCS Debugging:\n%s\n\nEnd GCS Debugging", strings.TrimSpace(out.String()))
+//}

vendor/github.com/Microsoft/hcsshim/internal/lcow/process.go

@@ -0,0 +1,161 @@
+package lcow
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/copywithtimeout"
+	"github.com/Microsoft/hcsshim/internal/hcs"
+	"github.com/Microsoft/hcsshim/internal/schema2"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// ByteCounts are the number of bytes copied to/from standard handles. Note
+// this is int64 rather than uint64 to match the golang io.Copy() signature.
+type ByteCounts struct {
+	In  int64
+	Out int64
+	Err int64
+}
+
+// ProcessOptions are the set of options which are passed to CreateProcessEx() to
+// create a utility vm.
+type ProcessOptions struct {
+	HCSSystem         *hcs.System
+	Process           *specs.Process
+	Stdin             io.Reader     // Optional reader for sending on to the processes stdin stream
+	Stdout            io.Writer     // Optional writer for returning the processes stdout stream
+	Stderr            io.Writer     // Optional writer for returning the processes stderr stream
+	CopyTimeout       time.Duration // Timeout for the copy
+	CreateInUtilityVm bool          // If the compute system is a utility VM
+	ByteCounts        ByteCounts    // How much data to copy on each stream if they are supplied. 0 means to io.EOF.
+}
+
+// CreateProcess creates a process either in an LCOW utility VM, or for starting
+// the init process. TODO: Potentially extend for exec'd processes.
+//
+// It's essentially a glorified wrapper around hcs.ComputeSystem CreateProcess used
+// for internal purposes.
+//
+// This is used on LCOW to run processes for remote filesystem commands, utilities,
+// and debugging.
+//
+// It optional performs IO copies with timeout between the pipes provided as input,
+// and the pipes in the process.
+//
+// In the ProcessOptions structure, if byte-counts are non-zero, a maximum of those
+// bytes are copied to the appropriate standard IO reader/writer. When zero,
+// it copies until EOF. It also returns byte-counts indicating how much data
+// was sent/received from the process.
+//
+// It is the responsibility of the caller to call Close() on the process returned.
+
+func CreateProcess(opts *ProcessOptions) (*hcs.Process, *ByteCounts, error) {
+
+	var environment = make(map[string]string)
+	copiedByteCounts := &ByteCounts{}
+
+	if opts == nil {
+		return nil, nil, fmt.Errorf("no options supplied")
+	}
+
+	if opts.HCSSystem == nil {
+		return nil, nil, fmt.Errorf("no HCS system supplied")
+	}
+
+	if opts.CreateInUtilityVm && opts.Process == nil {
+		return nil, nil, fmt.Errorf("process must be supplied for UVM process")
+	}
+
+	// Don't pass a process in if this is an LCOW container. This will start the init process.
+	if opts.Process != nil {
+		for _, v := range opts.Process.Env {
+			s := strings.SplitN(v, "=", 2)
+			if len(s) == 2 && len(s[1]) > 0 {
+				environment[s[0]] = s[1]
+			}
+		}
+		if _, ok := environment["PATH"]; !ok {
+			environment["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:"
+		}
+	}
+
+	processConfig := &ProcessParameters{
+		ProcessParameters: hcsschema.ProcessParameters{
+			CreateStdInPipe:  (opts.Stdin != nil),
+			CreateStdOutPipe: (opts.Stdout != nil),
+			CreateStdErrPipe: (opts.Stderr != nil),
+			EmulateConsole:   false,
+		},
+		CreateInUtilityVm: opts.CreateInUtilityVm,
+	}
+
+	if opts.Process != nil {
+		processConfig.Environment = environment
+		processConfig.CommandLine = strings.Join(opts.Process.Args, " ")
+		processConfig.WorkingDirectory = opts.Process.Cwd
+		if processConfig.WorkingDirectory == "" {
+			processConfig.WorkingDirectory = `/`
+		}
+	}
+
+	proc, err := opts.HCSSystem.CreateProcess(processConfig)
+	if err != nil {
+		logrus.Debugf("failed to create process: %s", err)
+		return nil, nil, err
+	}
+
+	processStdin, processStdout, processStderr, err := proc.Stdio()
+	if err != nil {
+		proc.Kill() // Should this have a timeout?
+		proc.Close()
+		return nil, nil, fmt.Errorf("failed to get stdio pipes for process %+v: %s", processConfig, err)
+	}
+
+	// Send the data into the process's stdin
+	if opts.Stdin != nil {
+		if copiedByteCounts.In, err = copywithtimeout.Copy(processStdin,
+			opts.Stdin,
+			opts.ByteCounts.In,
+			"stdin",
+			opts.CopyTimeout); err != nil {
+			return nil, nil, err
+		}
+
+		// Don't need stdin now we've sent everything. This signals GCS that we are finished sending data.
+		if err := proc.CloseStdin(); err != nil && !hcs.IsNotExist(err) && !hcs.IsAlreadyClosed(err) {
+			// This error will occur if the compute system is currently shutting down
+			if perr, ok := err.(*hcs.ProcessError); ok && perr.Err != hcs.ErrVmcomputeOperationInvalidState {
+				return nil, nil, err
+			}
+		}
+	}
+
+	// Copy the data back from stdout
+	if opts.Stdout != nil {
+		// Copy the data over to the writer.
+		if copiedByteCounts.Out, err = copywithtimeout.Copy(opts.Stdout,
+			processStdout,
+			opts.ByteCounts.Out,
+			"stdout",
+			opts.CopyTimeout); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	// Copy the data back from stderr
+	if opts.Stderr != nil {
+		// Copy the data over to the writer.
+		if copiedByteCounts.Err, err = copywithtimeout.Copy(opts.Stderr,
+			processStderr,
+			opts.ByteCounts.Err,
+			"stderr",
+			opts.CopyTimeout); err != nil {
+			return nil, nil, err
+		}
+	}
+	return proc, copiedByteCounts, nil
+}

vendor/github.com/Microsoft/hcsshim/internal/lcow/scratch.go

@@ -0,0 +1,168 @@
+package lcow
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/Microsoft/go-winio/vhd"
+	"github.com/Microsoft/hcsshim/internal/copyfile"
+	"github.com/Microsoft/hcsshim/internal/timeout"
+	"github.com/Microsoft/hcsshim/internal/uvm"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// CreateScratch uses a utility VM to create an empty scratch disk of a requested size.
+// It has a caching capability. If the cacheFile exists, and the request is for a default
+// size, a copy of that is made to the target. If the size is non-default, or the cache file
+// does not exist, it uses a utility VM to create target. It is the responsibility of the
+// caller to synchronise simultaneous attempts to create the cache file.
+func CreateScratch(lcowUVM *uvm.UtilityVM, destFile string, sizeGB uint32, cacheFile string, vmID string) error {
+
+	if lcowUVM == nil {
+		return fmt.Errorf("no uvm")
+	}
+
+	if lcowUVM.OS() != "linux" {
+		return fmt.Errorf("CreateLCOWScratch requires a linux utility VM to operate!")
+	}
+
+	// Smallest we can accept is the default scratch size as we can't size down, only expand.
+	if sizeGB < DefaultScratchSizeGB {
+		sizeGB = DefaultScratchSizeGB
+	}
+
+	logrus.Debugf("hcsshim::CreateLCOWScratch: Dest:%s size:%dGB cache:%s", destFile, sizeGB, cacheFile)
+
+	// Retrieve from cache if the default size and already on disk
+	if cacheFile != "" && sizeGB == DefaultScratchSizeGB {
+		if _, err := os.Stat(cacheFile); err == nil {
+			if err := copyfile.CopyFile(cacheFile, destFile, false); err != nil {
+				return fmt.Errorf("failed to copy cached file '%s' to '%s': %s", cacheFile, destFile, err)
+			}
+			logrus.Debugf("hcsshim::CreateLCOWScratch: %s fulfilled from cache (%s)", destFile, cacheFile)
+			return nil
+		}
+	}
+
+	// Create the VHDX
+	if err := vhd.CreateVhdx(destFile, sizeGB, defaultVhdxBlockSizeMB); err != nil {
+		return fmt.Errorf("failed to create VHDx %s: %s", destFile, err)
+	}
+
+	controller, lun, err := lcowUVM.AddSCSI(destFile, "", false) // No destination as not formatted
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("hcsshim::CreateLCOWScratch: %s at C=%d L=%d", destFile, controller, lun)
+
+	// Validate /sys/bus/scsi/devices/C:0:0:L exists as a directory
+
+	startTime := time.Now()
+	for {
+		testdCommand := []string{"test", "-d", fmt.Sprintf("/sys/bus/scsi/devices/%d:0:0:%d", controller, lun)}
+		testdProc, _, err := CreateProcess(&ProcessOptions{
+			HCSSystem:         lcowUVM.ComputeSystem(),
+			CreateInUtilityVm: true,
+			CopyTimeout:       timeout.ExternalCommandToStart,
+			Process:           &specs.Process{Args: testdCommand},
+		})
+		if err != nil {
+			lcowUVM.RemoveSCSI(destFile)
+			return fmt.Errorf("failed to run %+v following hot-add %s to utility VM: %s", testdCommand, destFile, err)
+		}
+		defer testdProc.Close()
+
+		testdProc.WaitTimeout(timeout.ExternalCommandToComplete)
+		testdExitCode, err := testdProc.ExitCode()
+		if err != nil {
+			lcowUVM.RemoveSCSI(destFile)
+			return fmt.Errorf("failed to get exit code from from %+v following hot-add %s to utility VM: %s", testdCommand, destFile, err)
+		}
+		if testdExitCode != 0 {
+			currentTime := time.Now()
+			elapsedTime := currentTime.Sub(startTime)
+			if elapsedTime > timeout.TestDRetryLoop {
+				lcowUVM.RemoveSCSI(destFile)
+				return fmt.Errorf("`%+v` return non-zero exit code (%d) following hot-add %s to utility VM", testdCommand, testdExitCode, destFile)
+			}
+		} else {
+			break
+		}
+		time.Sleep(time.Millisecond * 10)
+	}
+
+	// Get the device from under the block subdirectory by doing a simple ls. This will come back as (eg) `sda`
+	var lsOutput bytes.Buffer
+	lsCommand := []string{"ls", fmt.Sprintf("/sys/bus/scsi/devices/%d:0:0:%d/block", controller, lun)}
+	lsProc, _, err := CreateProcess(&ProcessOptions{
+		HCSSystem:         lcowUVM.ComputeSystem(),
+		CreateInUtilityVm: true,
+		CopyTimeout:       timeout.ExternalCommandToStart,
+		Process:           &specs.Process{Args: lsCommand},
+		Stdout:            &lsOutput,
+	})
+
+	if err != nil {
+		lcowUVM.RemoveSCSI(destFile)
+		return fmt.Errorf("failed to `%+v` following hot-add %s to utility VM: %s", lsCommand, destFile, err)
+	}
+	defer lsProc.Close()
+	lsProc.WaitTimeout(timeout.ExternalCommandToComplete)
+	lsExitCode, err := lsProc.ExitCode()
+	if err != nil {
+		lcowUVM.RemoveSCSI(destFile)
+		return fmt.Errorf("failed to get exit code from `%+v` following hot-add %s to utility VM: %s", lsCommand, destFile, err)
+	}
+	if lsExitCode != 0 {
+		lcowUVM.RemoveSCSI(destFile)
+		return fmt.Errorf("`%+v` return non-zero exit code (%d) following hot-add %s to utility VM", lsCommand, lsExitCode, destFile)
+	}
+	device := fmt.Sprintf(`/dev/%s`, strings.TrimSpace(lsOutput.String()))
+	logrus.Debugf("hcsshim: CreateExt4Vhdx: %s: device at %s", destFile, device)
+
+	// Format it ext4
+	mkfsCommand := []string{"mkfs.ext4", "-q", "-E", "lazy_itable_init=1", "-O", `^has_journal,sparse_super2,uninit_bg,^resize_inode`, device}
+	var mkfsStderr bytes.Buffer
+	mkfsProc, _, err := CreateProcess(&ProcessOptions{
+		HCSSystem:         lcowUVM.ComputeSystem(),
+		CreateInUtilityVm: true,
+		CopyTimeout:       timeout.ExternalCommandToStart,
+		Process:           &specs.Process{Args: mkfsCommand},
+		Stderr:            &mkfsStderr,
+	})
+	if err != nil {
+		lcowUVM.RemoveSCSI(destFile)
+		return fmt.Errorf("failed to `%+v` following hot-add %s to utility VM: %s", mkfsCommand, destFile, err)
+	}
+	defer mkfsProc.Close()
+	mkfsProc.WaitTimeout(timeout.ExternalCommandToComplete)
+	mkfsExitCode, err := mkfsProc.ExitCode()
+	if err != nil {
+		lcowUVM.RemoveSCSI(destFile)
+		return fmt.Errorf("failed to get exit code from `%+v` following hot-add %s to utility VM: %s", mkfsCommand, destFile, err)
+	}
+	if mkfsExitCode != 0 {
+		lcowUVM.RemoveSCSI(destFile)
+		return fmt.Errorf("`%+v` return non-zero exit code (%d) following hot-add %s to utility VM: %s", mkfsCommand, mkfsExitCode, destFile, strings.TrimSpace(mkfsStderr.String()))
+	}
+
+	// Hot-Remove before we copy it
+	if err := lcowUVM.RemoveSCSI(destFile); err != nil {
+		return fmt.Errorf("failed to hot-remove: %s", err)
+	}
+
+	// Populate the cache.
+	if cacheFile != "" && (sizeGB == DefaultScratchSizeGB) {
+		if err := copyfile.CopyFile(destFile, cacheFile, true); err != nil {
+			return fmt.Errorf("failed to seed cache '%s' from '%s': %s", destFile, cacheFile, err)
+		}
+	}
+
+	logrus.Debugf("hcsshim::CreateLCOWScratch: %s created (non-cache)", destFile)
+	return nil
+}

vendor/github.com/Microsoft/hcsshim/internal/lcow/tar2vhd.go

@@ -0,0 +1,46 @@
+package lcow
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/Microsoft/hcsshim/internal/uvm"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/sirupsen/logrus"
+)
+
+// TarToVhd streams a tarstream contained in an io.Reader to a fixed vhd file
+func TarToVhd(lcowUVM *uvm.UtilityVM, targetVHDFile string, reader io.Reader) (int64, error) {
+	logrus.Debugf("hcsshim: TarToVhd: %s", targetVHDFile)
+
+	if lcowUVM == nil {
+		return 0, fmt.Errorf("no utility VM passed")
+	}
+
+	//defer uvm.DebugLCOWGCS()
+
+	outFile, err := os.Create(targetVHDFile)
+	if err != nil {
+		return 0, fmt.Errorf("tar2vhd failed to create %s: %s", targetVHDFile, err)
+	}
+	defer outFile.Close()
+	// BUGBUG Delete the file on failure
+
+	tar2vhd, byteCounts, err := CreateProcess(&ProcessOptions{
+		HCSSystem:         lcowUVM.ComputeSystem(),
+		Process:           &specs.Process{Args: []string{"tar2vhd"}},
+		CreateInUtilityVm: true,
+		Stdin:             reader,
+		Stdout:            outFile,
+		CopyTimeout:       2 * time.Minute,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("failed to start tar2vhd for %s: %s", targetVHDFile, err)
+	}
+	defer tar2vhd.Close()
+
+	logrus.Debugf("hcsshim: TarToVhd: %s created, %d bytes", targetVHDFile, byteCounts.Out)
+	return byteCounts.Out, err
+}

vendor/github.com/Microsoft/hcsshim/internal/lcow/types.go

@@ -0,0 +1,11 @@
+package lcow
+
+import "github.com/Microsoft/hcsshim/internal/schema2"
+
+// Additional fields to hcsschema.ProcessParameters used by LCOW
+type ProcessParameters struct {
+	hcsschema.ProcessParameters
+
+	CreateInUtilityVm bool        `json:",omitempty"`
+	OCIProcess        interface{} `json:"OciProcess,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/lcow/vhd2tar.go

@@ -0,0 +1,75 @@
+package lcow
+
+import (
+	"fmt"
+	"io"
+	//	"os"
+
+	"github.com/Microsoft/hcsshim/internal/uvm"
+	//	specs "github.com/opencontainers/runtime-spec/specs-go"
+	//	"github.com/sirupsen/logrus"
+)
+
+// VhdToTar does what is says - it exports a VHD in a specified
+// folder (either a read-only layer.vhd, or a read-write scratch vhdx) to a
+// ReadCloser containing a tar-stream of the layers contents.
+func VhdToTar(lcowUVM *uvm.UtilityVM, vhdFile string, uvmMountPath string, isContainerScratch bool, vhdSize int64) (io.ReadCloser, error) {
+	return nil, fmt.Errorf("not implemented yet")
+	//	logrus.Debugf("hcsshim: VhdToTar: %s isScratch: %t", vhdFile, isContainerScratch)
+
+	//	if lcowUVM == nil {
+	//		return nil, fmt.Errorf("cannot VhdToTar as no utility VM is in configuration")
+	//	}
+
+	//	//defer uvm.DebugLCOWGCS()
+
+	//	vhdHandle, err := os.Open(vhdFile)
+	//	if err != nil {
+	//		return nil, fmt.Errorf("hcsshim: VhdToTar: failed to open %s: %s", vhdFile, err)
+	//	}
+	//	defer vhdHandle.Close()
+	//	logrus.Debugf("hcsshim: VhdToTar: exporting %s, size %d, isScratch %t", vhdHandle.Name(), vhdSize, isContainerScratch)
+
+	//	// Different binary depending on whether a RO layer or a RW scratch
+	//	command := "vhd2tar"
+	//	if isContainerScratch {
+	//		command = fmt.Sprintf("exportSandbox -path %s", uvmMountPath)
+	//	}
+
+	//	//	tar2vhd, byteCounts, err := lcowUVM.CreateProcess(&uvm.ProcessOptions{
+	//	//		Process: &specs.Process{Args: []string{"tar2vhd"}},
+	//	//		Stdin:   reader,
+	//	//		Stdout:  outFile,
+	//	//	})
+
+	//	// Start the binary in the utility VM
+	//	proc, stdin, stdout, _, err := config.createLCOWUVMProcess(command)
+	//	if err != nil {
+	//		return nil, fmt.Errorf("hcsshim: VhdToTar: %s: failed to create utils process %s: %s", vhdHandle.Name(), command, err)
+	//	}
+
+	//	if !isContainerScratch {
+	//		// Send the VHD contents to the utility VM processes stdin handle if not a container scratch
+	//		logrus.Debugf("hcsshim: VhdToTar: copying the layer VHD into the utility VM")
+	//		if _, err = copyWithTimeout(stdin, vhdHandle, vhdSize, processOperationTimeoutSeconds, fmt.Sprintf("vhdtotarstream: sending %s to %s", vhdHandle.Name(), command)); err != nil {
+	//			proc.Close()
+	//			return nil, fmt.Errorf("hcsshim: VhdToTar: %s: failed to copyWithTimeout on the stdin pipe (to utility VM): %s", vhdHandle.Name(), err)
+	//		}
+	//	}
+
+	//	// Start a goroutine which copies the stdout (ie the tar stream)
+	//	reader, writer := io.Pipe()
+	//	go func() {
+	//		defer writer.Close()
+	//		defer proc.Close()
+	//		logrus.Debugf("hcsshim: VhdToTar: copying tar stream back from the utility VM")
+	//		bytes, err := copyWithTimeout(writer, stdout, vhdSize, processOperationTimeoutSeconds, fmt.Sprintf("vhdtotarstream: copy tarstream from %s", command))
+	//		if err != nil {
+	//			logrus.Errorf("hcsshim: VhdToTar: %s:  copyWithTimeout on the stdout pipe (from utility VM) failed: %s", vhdHandle.Name(), err)
+	//		}
+	//		logrus.Debugf("hcsshim: VhdToTar: copied %d bytes of the tarstream of %s from the utility VM", bytes, vhdHandle.Name())
+	//	}()
+
+	//	// Return the read-side of the pipe connected to the goroutine which is reading from the stdout of the process in the utility VM
+	//	return reader, nil
+}