Enhance the clean script

(cherry picked from commit 4907fc9a4d)
Make the logs path of system-docker configurable
2018-10-17 14:34:33 +08:00 · 2018-10-17 13:24:46 +08:00 · 2018-10-17 13:24:46 +08:00 · 2018-10-15 17:45:51 +08:00 · 2018-10-14 11:07:22 +08:00
8 changed files with 241 additions and 206 deletions
--- a/Dockerfile.dapper
+++ b/Dockerfile.dapper
@@ -145,7 +145,7 @@ RUN curl -pfL ${SELINUX_POLICY_URL} > ${DOWNLOADS}/$(basename ${SELINUX_POLICY_U

 # Install Go
 RUN wget -O - https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${GOARCH}.tar.gz | tar -xzf - -C /usr/local && \
-    go get github.com/rancher/trash && go get github.com/golang/lint/golint
+    go get github.com/rancher/trash && go get golang.org/x/lint/golint

 # Install Host Docker
 RUN curl -fL ${!BUILD_DOCKER_URL} > /usr/bin/docker && \
--- a/cmd/control/install.go
+++ b/cmd/control/install.go
@@ -606,12 +606,42 @@ func seedData(baseName, cloudData string, files []string) error {
 		return err
 	}

-	if err = os.MkdirAll(filepath.Join(baseName, "/var/lib/rancher/conf/cloud-config.d"), 0700); err != nil {
+	stateSeedDir := "state_seed"
+	cloudConfigBase := "/var/lib/rancher/conf/cloud-config.d"
+	cloudConfigDir := ""
+
+	// If there is a separate boot partition, cloud-config should be written to RANCHER_STATE partition.
+	bootPartition, _, err := util.Blkid("RANCHER_BOOT")
+	if err != nil {
+		log.Errorf("Failed to run blkid: %s", err)
+	}
+	if bootPartition != "" {
+		stateSeedFullPath := filepath.Join(baseName, stateSeedDir)
+		if err = os.MkdirAll(stateSeedFullPath, 0700); err != nil {
+			return err
+		}
+
+		defer util.Unmount(stateSeedFullPath)
+
+		statePartition := install.GetStatePartition()
+		cmd := exec.Command("mount", statePartition, stateSeedFullPath)
+		//cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+		log.Debugf("seedData: mount %s to %s", statePartition, stateSeedFullPath)
+		if err = cmd.Run(); err != nil {
+			return err
+		}
+
+		cloudConfigDir = filepath.Join(baseName, stateSeedDir, cloudConfigBase)
+	} else {
+		cloudConfigDir = filepath.Join(baseName, cloudConfigBase)
+	}
+
+	if err = os.MkdirAll(cloudConfigDir, 0700); err != nil {
 		return err
 	}

 	if !strings.HasSuffix(cloudData, "empty.yml") {
-		if err = dfs.CopyFile(cloudData, baseName+"/var/lib/rancher/conf/cloud-config.d/", filepath.Base(cloudData)); err != nil {
+		if err = dfs.CopyFile(cloudData, cloudConfigDir, filepath.Base(cloudData)); err != nil {
 			return err
 		}
 	}
--- a/cmd/control/install/install.go
+++ b/cmd/control/install/install.go
@@ -46,7 +46,6 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string,
 		//rootfs := partition
 		// Don't use ResolveDevice - it can fail, whereas `blkid -L LABEL` works more often

-		cfg := config.LoadConfig()
 		d, _, err := util.Blkid("RANCHER_BOOT")
 		if err != nil {
 			log.Errorf("Failed to run blkid: %s", err)
@@ -55,18 +54,7 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string,
 			partition = d
 			baseName = filepath.Join(baseName, BootDir)
 		} else {
-			if dev := util.ResolveDevice(cfg.Rancher.State.Dev); dev != "" {
-				// try the rancher.state.dev setting
-				partition = dev
-			} else {
-				d, _, err := util.Blkid("RANCHER_STATE")
-				if err != nil {
-					log.Errorf("Failed to run blkid: %s", err)
-				}
-				if d != "" {
-					partition = d
-				}
-			}
+			partition = GetStatePartition()
 		}
 		cmd := exec.Command("lsblk", "-no", "pkname", partition)
 		log.Debugf("Run(%v)", cmd)
@@ -82,3 +70,17 @@ func MountDevice(baseName, device, partition string, raw bool) (string, string,
 	log.Debugf("mountdevice return2 -> d: %s, p: %s", device, partition)
 	return device, partition, cmd.Run()
 }
+
+func GetStatePartition() string {
+	cfg := config.LoadConfig()
+
+	if dev := util.ResolveDevice(cfg.Rancher.State.Dev); dev != "" {
+		// try the rancher.state.dev setting
+		return dev
+	}
+	d, _, err := util.Blkid("RANCHER_STATE")
+	if err != nil {
+		log.Errorf("Failed to run blkid: %s", err)
+	}
+	return d
+}
--- a/config/schema.go
+++ b/config/schema.go
@@ -1,209 +1,210 @@
 package config

 var schema = `{
-  "type": "object",
-  "additionalProperties": false,
+	"type": "object",
+	"additionalProperties": false,

-  "properties": {
-    "ssh_authorized_keys": {"$ref": "#/definitions/list_of_strings"},
-    "write_files": {
-      "type": "array",
-      "items": {"$ref": "#/definitions/file_config"}
-    },
-    "hostname": {"type": "string"},
-    "mounts": {"type": "array"},
-    "rancher": {"$ref": "#/definitions/rancher_config"},
-    "runcmd": {"type": "array"},
-    "bootcmd": {"type": "array"}
-  },
+	"properties": {
+		"ssh_authorized_keys": {"$ref": "#/definitions/list_of_strings"},
+		"write_files": {
+			"type": "array",
+			"items": {"$ref": "#/definitions/file_config"}
+		},
+		"hostname": {"type": "string"},
+		"mounts": {"type": "array"},
+		"rancher": {"$ref": "#/definitions/rancher_config"},
+		"runcmd": {"type": "array"},
+		"bootcmd": {"type": "array"}
+	},

-  "definitions": {
-    "rancher_config": {
-      "id": "#/definitions/rancher_config",
-      "type": "object",
-      "additionalProperties": false,
+	"definitions": {
+		"rancher_config": {
+			"id": "#/definitions/rancher_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "console": {"type": "string"},
-        "environment": {"type": "object"},
-        "cloud_init_services": {"type": "object"},
-        "services": {"type": "object"},
-        "bootstrap": {"type": "object"},
-        "autoformat": {"type": "object"},
-        "bootstrap_docker": {"$ref": "#/definitions/docker_config"},
-        "cloud_init": {"$ref": "#/definitions/cloud_init_config"},
-        "debug": {"type": "boolean"},
-        "rm_usr": {"type": "boolean"},
-        "no_sharedroot": {"type": "boolean"},
-        "log": {"type": "boolean"},
-        "force_console_rebuild": {"type": "boolean"},
-        "recovery": {"type": "boolean"},
-        "disable": {"$ref": "#/definitions/list_of_strings"},
-        "services_include": {"type": "object"},
-        "modules": {"$ref": "#/definitions/list_of_strings"},
-        "network": {"$ref": "#/definitions/network_config"},
-        "default_network": {"type": "object"},
-        "repositories": {"type": "object"},
-        "ssh": {"$ref": "#/definitions/ssh_config"},
-        "state": {"$ref": "#/definitions/state_config"},
-        "system_docker": {"$ref": "#/definitions/docker_config"},
-        "upgrade": {"$ref": "#/definitions/upgrade_config"},
-        "docker": {"$ref": "#/definitions/docker_config"},
-        "registry_auths": {"type": "object"},
-        "defaults": {"$ref": "#/definitions/defaults_config"},
-        "resize_device": {"type": "string"},
-        "sysctl": {"type": "object"},
-        "restart_services": {"type": "array"},
-	"hypervisor_service": {"type": "boolean"},
-        "shutdown_timeout": {"type": "integer"},
-	"preload_wait": {"type": "boolean"}
-      }
-    },
+			"properties": {
+				"console": {"type": "string"},
+				"environment": {"type": "object"},
+				"cloud_init_services": {"type": "object"},
+				"services": {"type": "object"},
+				"bootstrap": {"type": "object"},
+				"autoformat": {"type": "object"},
+				"bootstrap_docker": {"$ref": "#/definitions/docker_config"},
+				"cloud_init": {"$ref": "#/definitions/cloud_init_config"},
+				"debug": {"type": "boolean"},
+				"rm_usr": {"type": "boolean"},
+				"no_sharedroot": {"type": "boolean"},
+				"log": {"type": "boolean"},
+				"force_console_rebuild": {"type": "boolean"},
+				"recovery": {"type": "boolean"},
+				"disable": {"$ref": "#/definitions/list_of_strings"},
+				"services_include": {"type": "object"},
+				"modules": {"$ref": "#/definitions/list_of_strings"},
+				"network": {"$ref": "#/definitions/network_config"},
+				"default_network": {"type": "object"},
+				"repositories": {"type": "object"},
+				"ssh": {"$ref": "#/definitions/ssh_config"},
+				"state": {"$ref": "#/definitions/state_config"},
+				"system_docker": {"$ref": "#/definitions/docker_config"},
+				"upgrade": {"$ref": "#/definitions/upgrade_config"},
+				"docker": {"$ref": "#/definitions/docker_config"},
+				"registry_auths": {"type": "object"},
+				"defaults": {"$ref": "#/definitions/defaults_config"},
+				"resize_device": {"type": "string"},
+				"sysctl": {"type": "object"},
+				"restart_services": {"type": "array"},
+				"hypervisor_service": {"type": "boolean"},
+				"shutdown_timeout": {"type": "integer"},
+				"preload_wait": {"type": "boolean"}
+			}
+		},

-    "file_config": {
-      "id": "#/definitions/file_config",
-      "type": "object",
-      "additionalProperties": false,
+		"file_config": {
+			"id": "#/definitions/file_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "encoding": {"type": "string"},
-        "container": {"type": "string"},
-        "content": {"type": "string"},
-        "owner": {"type": "string"},
-        "path": {"type": "string"},
-        "permissions": {"type": "string"}
-      }
-    },
+			"properties": {
+				"encoding": {"type": "string"},
+				"container": {"type": "string"},
+				"content": {"type": "string"},
+				"owner": {"type": "string"},
+				"path": {"type": "string"},
+				"permissions": {"type": "string"}
+			}
+		},

-    "network_config": {
-      "id": "#/definitions/network_config",
-      "type": "object",
-      "additionalProperties": false,
+		"network_config": {
+			"id": "#/definitions/network_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "pre_cmds": {"$ref": "#/definitions/list_of_strings"},
-        "dns": {"type": "object"},
-        "interfaces": {"type": "object"},
-        "post_cmds": {"$ref": "#/definitions/list_of_strings"},
-        "http_proxy": {"type": "string"},
-        "https_proxy": {"type": "string"},
-        "no_proxy": {"type": "string"}
-      }
-    },
+			"properties": {
+				"pre_cmds": {"$ref": "#/definitions/list_of_strings"},
+				"dns": {"type": "object"},
+				"interfaces": {"type": "object"},
+				"post_cmds": {"$ref": "#/definitions/list_of_strings"},
+				"http_proxy": {"type": "string"},
+				"https_proxy": {"type": "string"},
+				"no_proxy": {"type": "string"}
+			}
+		},

-    "upgrade_config": {
-      "id": "#/definitions/upgrade_config",
-      "type": "object",
-      "additionalProperties": false,
+		"upgrade_config": {
+			"id": "#/definitions/upgrade_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "url": {"type": "string"},
-        "image": {"type": "string"},
-        "rollback": {"type": "string"}
-      }
-    },
+			"properties": {
+				"url": {"type": "string"},
+				"image": {"type": "string"},
+				"rollback": {"type": "string"}
+			}
+		},

-    "docker_config": {
-      "id": "#/definitions/docker_config",
-      "type": "object",
-      "additionalProperties": false,
+		"docker_config": {
+			"id": "#/definitions/docker_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "engine": {"type": "string"},
-        "tls": {"type": "boolean"},
-        "tls_args": {"$ref": "#/definitions/list_of_strings"},
-        "args": {"$ref": "#/definitions/list_of_strings"},
-        "extra_args": {"$ref": "#/definitions/list_of_strings"},
-        "server_cert": {"type": "string"},
-        "server_key": {"type": "string"},
-        "ca_cert": {"type": "string"},
-        "ca_key": {"type": "string"},
-        "environment": {"$ref": "#/definitions/list_of_strings"},
-        "storage_context": {"type": "string"},
-        "exec": {"type": ["boolean", "null"]},
-        "bridge": {"type": "string"},
-        "bip": {"type": "string"},
-        "config_file": {"type": "string"},
-        "containerd": {"type": "string"},
-        "debug": {"type": ["boolean", "null"]},
-        "exec_root": {"type": "string"},
-        "group": {"type": "string"},
-        "graph": {"type": "string"},
-        "host": {"type": "array"},
-        "live_restore": {"type": ["boolean", "null"]},
-        "log_driver": {"type": "string"},
-        "log_opts": {"type": "object"},
-        "pid_file": {"type": "string"},
-        "registry_mirror": {"type": "string"},
-        "restart": {"type": ["boolean", "null"]},
-        "selinux_enabled": {"type": ["boolean", "null"]},
-        "storage_driver": {"type": "string"},
-        "userland_proxy": {"type": ["boolean", "null"]},
-        "insecure_registry": {"$ref": "#/definitions/list_of_strings"}
-      }
-    },
+			"properties": {
+				"engine": {"type": "string"},
+				"tls": {"type": "boolean"},
+				"tls_args": {"$ref": "#/definitions/list_of_strings"},
+				"args": {"$ref": "#/definitions/list_of_strings"},
+				"extra_args": {"$ref": "#/definitions/list_of_strings"},
+				"server_cert": {"type": "string"},
+				"server_key": {"type": "string"},
+				"ca_cert": {"type": "string"},
+				"ca_key": {"type": "string"},
+				"environment": {"$ref": "#/definitions/list_of_strings"},
+				"storage_context": {"type": "string"},
+				"exec": {"type": ["boolean", "null"]},
+				"bridge": {"type": "string"},
+				"bip": {"type": "string"},
+				"config_file": {"type": "string"},
+				"containerd": {"type": "string"},
+				"debug": {"type": ["boolean", "null"]},
+				"exec_root": {"type": "string"},
+				"group": {"type": "string"},
+				"graph": {"type": "string"},
+				"host": {"type": "array"},
+				"live_restore": {"type": ["boolean", "null"]},
+				"log_driver": {"type": "string"},
+				"log_opts": {"type": "object"},
+				"pid_file": {"type": "string"},
+				"registry_mirror": {"type": "string"},
+				"restart": {"type": ["boolean", "null"]},
+				"selinux_enabled": {"type": ["boolean", "null"]},
+				"storage_driver": {"type": "string"},
+				"userland_proxy": {"type": ["boolean", "null"]},
+				"insecure_registry": {"$ref": "#/definitions/list_of_strings"}
+			}
+		},

-    "ssh_config": {
-      "id": "#/definitions/ssh_config",
-      "type": "object",
-      "additionalProperties": false,
+		"ssh_config": {
+			"id": "#/definitions/ssh_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "keys": {"type": "object"},
-        "daemon": {"type": "boolean"},
-        "port": {"type": "integer"},
-        "listen_address": {"type": "string"}
-      }
-    },
+			"properties": {
+				"keys": {"type": "object"},
+				"daemon": {"type": "boolean"},
+				"port": {"type": "integer"},
+				"listen_address": {"type": "string"}
+			}
+		},

-    "state_config": {
-      "id": "#/definitions/state_config",
-      "type": "object",
-      "additionalProperties": false,
+		"state_config": {
+			"id": "#/definitions/state_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "directory": {"type": "string"},
-        "fstype": {"type": "string"},
-        "dev": {"type": "string"},
-        "wait": {"type": "boolean"},
-        "required": {"type": "boolean"},
-        "autoformat": {"$ref": "#/definitions/list_of_strings"},
-        "mdadm_scan": {"type": "boolean"},
-        "rngd": {"type": "boolean"},
-        "script": {"type": "string"},
-        "oem_fstype": {"type": "string"},
-        "oem_dev": {"type": "string"}
-      }
-    },
+			"properties": {
+				"directory": {"type": "string"},
+				"fstype": {"type": "string"},
+				"dev": {"type": "string"},
+				"wait": {"type": "boolean"},
+				"required": {"type": "boolean"},
+				"autoformat": {"$ref": "#/definitions/list_of_strings"},
+				"mdadm_scan": {"type": "boolean"},
+				"rngd": {"type": "boolean"},
+				"script": {"type": "string"},
+				"oem_fstype": {"type": "string"},
+				"oem_dev": {"type": "string"}
+			}
+		},

-    "cloud_init_config": {
-      "id": "#/definitions/cloud_init_config",
-      "type": "object",
-      "additionalProperties": false,
+		"cloud_init_config": {
+			"id": "#/definitions/cloud_init_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "datasources": {"$ref": "#/definitions/list_of_strings"}
-      }
-    },
+			"properties": {
+				"datasources": {"$ref": "#/definitions/list_of_strings"}
+			}
+		},

-    "defaults_config": {
-      "id": "#/definitions/defaults_config",
-      "type": "object",
-      "additionalProperties": false,
+		"defaults_config": {
+			"id": "#/definitions/defaults_config",
+			"type": "object",
+			"additionalProperties": false,

-      "properties": {
-        "hostname": {"type": "string"},
-        "docker": {"type": "object"},
-        "network": {"$ref": "#/definitions/network_config"}
-      }
-    },
+			"properties": {
+				"hostname": {"type": "string"},
+				"docker": {"type": "object"},
+				"network": {"$ref": "#/definitions/network_config"},
+				"system_docker_logs": {"type": "string"}
+			}
+		},

-    "list_of_strings": {
-      "type": "array",
-      "items": {"type": "string"},
-      "uniqueItems": true
-    }
-  }
+		"list_of_strings": {
+			"type": "array",
+			"items": {"type": "string"},
+			"uniqueItems": true
+		}
+	}
 }

 `
--- a/config/types.go
+++ b/config/types.go
@@ -24,7 +24,6 @@ const (
 	ImagesPattern    = "images*.tar"
 	ModulesArchive   = "/modules.tar"
 	Debug            = false
-	SystemDockerLog  = "/var/log/system-docker.log"
 	SystemDockerBin  = "/usr/bin/system-dockerd"

 	HashLabel         = "io.rancher.os.hash"
@@ -211,9 +210,10 @@ type CloudInit struct {
 }

 type Defaults struct {
-	Hostname string                `yaml:"hostname,omitempty"`
-	Docker   DockerConfig          `yaml:"docker,omitempty"`
-	Network  netconf.NetworkConfig `yaml:"network,omitempty"`
+	Hostname         string                `yaml:"hostname,omitempty"`
+	Docker           DockerConfig          `yaml:"docker,omitempty"`
+	Network          netconf.NetworkConfig `yaml:"network,omitempty"`
+	SystemDockerLogs string                `yaml:"system_docker_logs,omitempty"`
 }

 func (r Repositories) ToArray() []string {
--- a/init/init.go
+++ b/init/init.go
@@ -183,7 +183,7 @@ func getLaunchConfig(cfg *config.CloudConfig, dockerCfg *config.DockerConfig) (*
 	launchConfig.Environment = dockerCfg.Environment

 	if !cfg.Rancher.Debug {
-		launchConfig.LogFile = config.SystemDockerLog
+		launchConfig.LogFile = cfg.Rancher.Defaults.SystemDockerLogs
 	}

 	return &launchConfig, args
--- a/os-config.tpl.yml
+++ b/os-config.tpl.yml
@@ -16,6 +16,7 @@ rancher:
    network:
      dns:
        nameservers: [8.8.8.8, 8.8.4.4]
+    system_docker_logs: /var/log/system-docker.log
  ssh:
    daemon: true
  hypervisor_service: true
--- a/scripts/clean
+++ b/scripts/clean
@@ -6,5 +6,6 @@ rm -rf build dist bin images/*/build state
 rm -rf ./scripts/images/openstack/dist
 rm -rf ./scripts/images/vmware/dist

-docker rmi $(docker images --format "{{.Repository}}:{{.Tag}}" | grep rancher) || true
-docker rmi $(docker images --filter dangling=true -q) ||true
+docker rmi $(docker images --format "{{.Repository}}:{{.Tag}}" | grep -E "rancher|openstack|kexec|os") || true
+docker rmi $(docker images --filter dangling=true -q) || true
+git clean -dxf
Author	SHA1	Message	Date
niusmallnan	b8c5d18584	Enhance the clean script (cherry picked from commit `4907fc9a4d`)	2018-10-17 14:34:33 +08:00
niusmallnan	e605f82ad8	Make the logs path of system-docker configurable	2018-10-17 13:24:46 +08:00
niusmallnan	370e5f68a9	Format config/schema.go file	2018-10-17 13:24:46 +08:00
niusmallnan	2383e04142	Fix a bug when installing ros to disk with noformat. The cloud-config files cannot written to RANCHER_STATE disk when users set the RANCHER_BOOT partition. (cherry picked from commit `ab4e0e590b`)	2018-10-15 17:45:51 +08:00
niusmallnan	e738a20901	Fix build error caused by golint (cherry picked from commit `c04293f6dd`)	2018-10-14 11:07:22 +08:00