From 80af9be2a86a4b7c7d696f1ba57684efcf4f2081 Mon Sep 17 00:00:00 2001 From: WangFengTu Date: Sun, 19 Jan 2020 11:25:09 +0800 Subject: [PATCH 38/40] Fix some devicemapper bugs Signed-off-by: WangFengTu --- .../storage/drivers/devmapper/deviceset.go | 74 +++++++++++++++++-- .../storage/drivers/devmapper/driver.go | 44 ++++++++--- .../storage/drivers/devmapper/sem_stat.go | 36 +++++++++ .../storage/pkg/devicemapper/devmapper.go | 65 ++++++++++++++-- .../pkg/devicemapper/devmapper_wrapper.go | 72 ++++++++++++++++++ vendor/github.com/containers/storage/store.go | 7 +- 6 files changed, 274 insertions(+), 24 deletions(-) create mode 100644 vendor/github.com/containers/storage/drivers/devmapper/sem_stat.go diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index b6f22e9..4ebad51 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -34,13 +34,16 @@ import ( ) var ( - defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 - defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 - defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 - defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors - defaultUdevSyncOverride = false - maxDeviceID = 0xffffff // 24 bit, pool limit - deviceIDMapSz = (maxDeviceID + 1) / 8 + defaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 + defaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 + defaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 + defaultThinpBlockSize uint32 = 128 // 64K = 128 512b sectors + defaultUdevSyncOverride = false + maxDeviceID = 0xffffff // 24 bit, pool limit + deviceIDMapSz = (maxDeviceID + 1) / 8 + // The default timeout for latest systemd on RTOS and EulerOS is 180s + // we use 185s to make sure the timeout really happened. + defaultUdevWaitTimeout = 185 driverDeferredRemovalSupport = false enableDeferredRemoval = false enableDeferredDeletion = false @@ -1409,6 +1412,8 @@ func (devices *DeviceSet) rollbackTransaction() error { devices.markDeviceIDFree(devices.DeviceID) } + delete(devices.Devices, dinfo.Hash) + if err := devices.removeTransactionMetaData(); err != nil { logrus.Errorf("devmapper: Unable to remove transaction meta file %s: %s", devices.transactionMetaFile(), err) } @@ -1746,6 +1751,35 @@ func (devices *DeviceSet) initDevmapper(doInit bool) (retErr error) { devices.devicePrefix = fmt.Sprintf("container-%d:%d-%d", major(st.Dev), minor(st.Dev), st.Ino) logrus.Debugf("devmapper: Generated prefix: %s", devices.devicePrefix) + deviceNames, err := devicemapper.GetDeviceList() + if err != nil { + logrus.Debugf("devmapper: Failed to get device list: %s", err) + } + + for _, name := range deviceNames { + if !strings.HasPrefix(name, devices.devicePrefix) { + continue + } + _, length, _, _, err := devicemapper.GetStatus(name) + if err != nil { + logrus.Warnf("devmapper: get device status(%s): %s", name, err) + continue + } + // remove broken device + if length == 0 { + if err := devicemapper.RemoveDevice(name); err != nil { + logrus.Warnf("devmapper: remove broken device(%s): %s", name, err) + } + logrus.Debugf("devmapper: remove broken device: %s", name) + } + if _, err := os.Stat(filepath.Join("/dev/mapper/", name)); err != nil { + if err := devicemapper.RemoveDevice(name); err != nil { + logrus.Warnf("devmapper: remove incompelete device(%s): %v", name, err) + } + logrus.Debugf("devmapper: remove incompelete device: %s", name) + } + } + // Check for the existence of the thin-pool device poolExists, err := devices.thinPoolExists(devices.getPoolName()) if err != nil { @@ -2161,6 +2195,13 @@ func (devices *DeviceSet) deactivateDeviceMode(info *devInfo, deferredRemove boo if deferredRemove { err = devicemapper.RemoveDeviceDeferred(info.Name()) + if err != nil { + if err == devicemapper.ErrEnxio { + logrus.Warnf("devmapper: device %s has gone", info.Name()) + return nil + } + return err + } } else { err = devices.removeDevice(info.Name()) } @@ -2445,6 +2486,18 @@ func (devices *DeviceSet) UnmountDevice(hash, mountPath string) error { } logrus.Debug("devmapper: Unmount done") + // Remove the mountpoint here. Removing the mountpoint (in newer kernels) + // will cause all other instances of this mount in other mount namespaces + // to be killed (this is an anti-DoS measure that is necessary for things + // like devicemapper). This is necessary to avoid cases where a libdm mount + // that is present in another namespace will cause subsequent RemoveDevice + // operations to fail. We ignore any errors here because this may fail on + // older kernels which don't have + // torvalds/linux@8ed936b5671bfb33d89bc60bdcc7cf0470ba52fe applied. + if err := os.Remove(mountPath); err != nil { + logrus.Debugf("devmapper: error doing a remove on unmounted device %s: %v", mountPath, err) + } + return devices.deactivateDevice(info) } @@ -2665,6 +2718,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ foundBlkDiscard := false var lvmSetupConfig directLVMConfig + udevWaitTimeout := int64(defaultUdevWaitTimeout) testMode := false for _, option := range options { key, val, err := parsers.ParseKeyValueOpt(option) @@ -2815,6 +2869,11 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ devicemapper.LogInit(devicemapper.DefaultLogger{ Level: int(level), }) + case "dm.udev_wait_timeout": + udevWaitTimeout, err = strconv.ParseInt(val, 10, 32) + if err != nil { + return nil, err + } case "test": testMode, err = strconv.ParseBool(val) if err != nil { @@ -2824,6 +2883,7 @@ func NewDeviceSet(root string, doInit bool, options []string, uidMaps, gidMaps [ return nil, fmt.Errorf("devmapper: Unknown option %s", key) } } + devicemapper.SetUdevWaitTimtout(udevWaitTimeout) if !testMode { if err := validateLVMConfig(lvmSetupConfig); err != nil { diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index 39a4fbe..f80643f 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -14,9 +14,10 @@ import ( "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" - "github.com/containers/storage/pkg/system" units "github.com/docker/go-units" + "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) func init() { @@ -40,10 +41,6 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - if err := mount.MakePrivate(home); err != nil { - return nil, err - } - d := &Driver{ DeviceSet: deviceSet, home: home, @@ -94,6 +91,14 @@ func (d *Driver) Status() [][2]string { if vStr, err := devicemapper.GetLibraryVersion(); err == nil { status = append(status, [2]string{"Library Version", vStr}) } + + usz, mni, err := checkSemSetStat() + status = append(status, [2]string{"Semaphore Set Used", fmt.Sprintf("%d", usz)}) + status = append(status, [2]string{"Semaphore Set Total", fmt.Sprintf("%d", mni)}) + if err != nil { + status = append(status, [2]string{"WARNING", fmt.Sprintf("%v", err)}) + } + return status } @@ -109,18 +114,25 @@ func (d *Driver) Metadata(id string) (map[string]string, error) { metadata["DeviceId"] = strconv.Itoa(m.deviceID) metadata["DeviceSize"] = strconv.FormatUint(m.deviceSize, 10) metadata["DeviceName"] = m.deviceName + metadata["MergedDir"] = path.Join(d.home, "mnt", id, "rootfs") return metadata, nil } // Cleanup unmounts a device. func (d *Driver) Cleanup() error { err := d.DeviceSet.Shutdown(d.home) + umountErr := mount.RecursiveUnmount(d.home) + + // in case we have two errors, prefer the one from Shutdown() + if err != nil { + return err + } - if err2 := mount.Unmount(d.home); err == nil { - err = err2 + if umountErr != nil { + return errors.Wrapf(umountErr, "error unmounting %s", d.home) } - return err + return nil } // CreateReadWrite creates a layer that is writable for use as a container @@ -158,7 +170,21 @@ func (d *Driver) Remove(id string) error { if err := d.DeviceSet.DeleteDevice(id, false); err != nil { return fmt.Errorf("failed to remove device %s: %v", id, err) } - return system.EnsureRemoveAll(path.Join(d.home, "mnt", id)) + + // Most probably the mount point is already removed on Put() + // (see DeviceSet.UnmountDevice()), but just in case it was not + // let's try to remove it here as well, ignoring errors as + // an older kernel can return EBUSY if e.g. the mount was leaked + // to other mount namespaces. A failure to remove the container's + // mount point is not important and should not be treated + // as a failure to remove the container. + mp := path.Join(d.home, "mnt", id) + err := unix.Rmdir(mp) + if err != nil && !os.IsNotExist(err) { + logrus.WithField("storage-driver", "devicemapper").Warnf("unable to remove mount point %q: %s", mp, err) + } + + return nil } // Get mounts a device with given id into the root filesystem diff --git a/vendor/github.com/containers/storage/drivers/devmapper/sem_stat.go b/vendor/github.com/containers/storage/drivers/devmapper/sem_stat.go new file mode 100644 index 0000000..2aa90c1 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/devmapper/sem_stat.go @@ -0,0 +1,36 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. +// iSulad-kit licensed under the Mulan PSL v1. +// You can use this software according to the terms and conditions of the Mulan PSL v1. +// You may obtain a copy of Mulan PSL v1 at: +// http://license.coscl.org.cn/MulanPSL +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v1 for more details. +// Description: iSulad image kit +// Author: liruilin +// Create: 2019-01-19 + +package devmapper + +/* +#include +#include +int mysemctl(int cmd, struct seminfo *p){ + return semctl(0, 0, cmd, p); +} +*/ +import "C" +import ( + "fmt" +) + +func checkSemSetStat() (int, int, error) { + var seminfo *C.struct_seminfo = new(C.struct_seminfo) + C.mysemctl(C.SEM_INFO, seminfo) + var err error = nil + if seminfo.semusz == seminfo.semmni { + err = fmt.Errorf("system semaphore nums has attached limit: %d", int(seminfo.semusz)) + } + return int(seminfo.semusz), int(seminfo.semmni), err +} diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper.go index 6a0ac24..48d760c 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "runtime" + "time" "unsafe" "github.com/sirupsen/logrus" @@ -54,11 +55,13 @@ var ( ErrTaskGetDeps = errors.New("dm_task_get_deps failed") ErrTaskGetInfo = errors.New("dm_task_get_info failed") ErrTaskGetDriverVersion = errors.New("dm_task_get_driver_version failed") + ErrTaskGetNames = errors.New("dm_task_get_names failed") ErrTaskDeferredRemove = errors.New("dm_task_deferred_remove failed") ErrTaskSetCookie = errors.New("dm_task_set_cookie failed") ErrNilCookie = errors.New("cookie ptr can't be nil") ErrGetBlockSize = errors.New("Can't get block size") ErrUdevWait = errors.New("wait on udev cookie failed") + ErrUdevWaitTimeout = errors.New("wait on udev cookie time out") ErrSetDevDir = errors.New("dm_set_dev_dir failed") ErrGetLibraryVersion = errors.New("dm_get_library_version failed") ErrCreateRemoveTask = errors.New("Can't create task of type deviceRemove") @@ -70,9 +73,11 @@ var ( ) var ( - dmSawBusy bool - dmSawExist bool - dmSawEnxio bool // No Such Device or Address + dmSawBusy bool + dmSawExist bool + dmSawEnxio bool // No Such Device or Address + dmSawEnoData bool // No data available + dmUdevWaitTimeout int64 ) type ( @@ -236,6 +241,14 @@ func (t *Task) getInfoWithDeferred() (*Info, error) { return info, nil } +func (t *Task) getDeviceList() ([]string, error) { + res := DmTaskGetNames(t.unmanaged) + if res == nil { + return nil, ErrTaskGetNames + } + return res, nil +} + func (t *Task) getDriverVersion() (string, error) { res := DmTaskGetDriverVersion(t.unmanaged) if res == "" { @@ -254,13 +267,36 @@ func (t *Task) getNextTarget(next unsafe.Pointer) (nextPtr unsafe.Pointer, start // UdevWait waits for any processes that are waiting for udev to complete the specified cookie. func UdevWait(cookie *uint) error { - if res := DmUdevWait(*cookie); res != 1 { - logrus.Debugf("devicemapper: Failed to wait on udev cookie %d, %d", *cookie, res) - return ErrUdevWait + chError := make(chan error) + go func() { + if res := DmUdevWait(*cookie); res != 1 { + logrus.Debugf("Failed to wait on udev cookie %d", *cookie) + chError <- ErrUdevWait + } + chError <- nil + }() + select { + case err := <-chError: + return err + case <-time.After(time.Second * time.Duration(dmUdevWaitTimeout)): + logrus.Errorf("Failed to wait on udev cookie %d: timeout %v", *cookie, dmUdevWaitTimeout) + if res := DmUdevComplete(*cookie); res != 1 { + // This is bad to return here + logrus.Errorf("Failed to complete udev cookie %d on udev wait timeout", *cookie) + return ErrUdevWaitTimeout + } + // wait DmUdevWait return after DmUdevComplete + <-chError + return ErrUdevWaitTimeout } return nil } +// SetUdevWaitTimtout sets udev wait timeout +func SetUdevWaitTimtout(t int64) { + dmUdevWaitTimeout = t +} + // SetDevDir sets the dev folder for the device mapper library (usually /dev). func SetDevDir(dir string) error { if res := DmSetDevDir(dir); res != 1 { @@ -543,6 +579,18 @@ func GetInfoWithDeferred(name string) (*Info, error) { return task.getInfoWithDeferred() } +// GetDevices get all device name +func GetDeviceList() ([]string, error) { + task := TaskCreate(deviceList) + if task == nil { + return nil, fmt.Errorf("devicemapper: Can't create deviceList task") + } + if err := task.run(); err != nil { + return nil, err + } + return task.getDeviceList() +} + // GetDriverVersion is the programmatic example of "dmsetup version". // It outputs version information of the driver. func GetDriverVersion() (string, error) { @@ -710,10 +758,15 @@ func DeleteDevice(poolName string, deviceID int) error { } dmSawBusy = false + dmSawEnoData = false if err := task.run(); err != nil { if dmSawBusy { return ErrBusy } + if dmSawEnoData { + logrus.Debugf("devicemapper: Device(id: %d) from pool(%s) does not exist", deviceID, poolName) + return nil + } return fmt.Errorf("devicemapper: Error running DeleteDevice %s", err) } return nil diff --git a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go index 190d83d..b7fd8b8 100644 --- a/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go +++ b/vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go @@ -6,6 +6,9 @@ package devicemapper #define _GNU_SOURCE #include #include // FIXME: present only for BLKGETSIZE64, maybe we can remove it? +#include +#include + // FIXME: Can't we find a way to do the logging in pure Go? extern void StorageDevmapperLogCallback(int level, char *file, int line, int dm_errno_or_class, char *str); @@ -32,6 +35,51 @@ static void log_with_errno_init() { dm_log_with_errno_init(log_cb); } + +// FIXME: how to use dm_task_get_names directly +static char **local_dm_task_get_names(struct dm_task *dmt, unsigned int *size) { + struct dm_names *ns, *ns1; + unsigned next = 0; + char **result; + int i = 0; + + if (!(ns = dm_task_get_names(dmt))) + return NULL; + + // No devices found + if (!ns->dev) + return NULL; + + // calucate the total devices + ns1 = ns; + *size = 0; + do { + ns1 = (struct dm_names *)((char *) ns1 + next); + (*size)++; + next = ns1->next; + } while (next); + + result = malloc(sizeof(char *)* (*size)); + if (!result) + return NULL; + + next = 0; + do { + ns = (struct dm_names *)((char *) ns + next); + result[i++] = strdup(ns->name); + next = ns->next; + } while (next); + + return result; +} + +void free_devices_names(char **names, unsigned int size) { + int i; + + for (i = 0; i < size; i++) + free(names[i]); + free(names); +} */ import "C" @@ -69,6 +117,7 @@ var ( DmTaskGetDeps = dmTaskGetDepsFct DmTaskGetInfo = dmTaskGetInfoFct DmTaskGetDriverVersion = dmTaskGetDriverVersionFct + DmTaskGetNames = dmTaskGetNamesFct DmTaskRun = dmTaskRunFct DmTaskSetAddNode = dmTaskSetAddNodeFct DmTaskSetCookie = dmTaskSetCookieFct @@ -77,6 +126,7 @@ var ( DmTaskSetRo = dmTaskSetRoFct DmTaskSetSector = dmTaskSetSectorFct DmUdevWait = dmUdevWaitFct + DmUdevComplete = dmUdevCompleteFct DmUdevSetSyncSupport = dmUdevSetSyncSupportFct DmUdevGetSyncSupport = dmUdevGetSyncSupportFct DmCookieSupported = dmCookieSupportedFct @@ -189,6 +239,24 @@ func dmTaskGetInfoFct(task *cdmTask, info *Info) int { return int(C.dm_task_get_info((*C.struct_dm_task)(task), &Cinfo)) } +func dmTaskGetNamesFct(task *cdmTask) []string { + var res []string + var names []*C.char + len := C.uint(0) + Cnames := C.local_dm_task_get_names((*C.struct_dm_task)(task), &len) + defer C.free_devices_names(Cnames, len) + + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&names)) + hdr.Cap = int(len) + hdr.Len = int(len) + hdr.Data = uintptr(unsafe.Pointer(Cnames)) + + for _, name := range names { + res = append(res, C.GoString(name)) + } + return res +} + func dmTaskGetDriverVersionFct(task *cdmTask) string { buffer := C.malloc(128) defer C.free(buffer) @@ -227,6 +295,10 @@ func dmUdevWaitFct(cookie uint) int { return int(C.dm_udev_wait(C.uint32_t(cookie))) } +func dmUdevCompleteFct(cookie uint) int { + return int(C.dm_udev_complete(C.uint32_t(cookie))) +} + func dmCookieSupportedFct() int { return int(C.dm_cookie_supported()) } diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index f346dc9..2ebd558 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2833,12 +2833,15 @@ func (s *store) Unmount(id string, force bool) (bool, error) { if err != nil { return false, err } + rlstore.Lock() - defer rlstore.Unlock() if modified, err := rlstore.Modified(); modified || err != nil { rlstore.Load() } - if rlstore.Exists(id) { + exist := rlstore.Exists(id) + rlstore.Unlock() + + if exist { return rlstore.Unmount(id, force) } return false, ErrLayerUnknown -- 2.19.1