@Override
  public synchronized List<PrivilegedOperation> preStart(Container container)
      throws ResourceHandlerException {
    String containerIdStr = container.getContainerId().toString();

    // Assign Gpus to container if requested some.
    GpuResourceAllocator.GpuAllocation allocation = gpuAllocator.assignGpus(
        container);

    // Create device cgroups for the container
    cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
        containerIdStr);
    if (!OCIContainerRuntime.isOCICompliantContainerRequested(
        nmContext.getConf(),
        container.getLaunchContext().getEnvironment())) {
      // Write to devices cgroup only for non-docker container. The reason is
      // docker engine runtime runc do the devices cgroups initialize in the
      // pre-hook, see:
      //   https://github.com/opencontainers/runc/blob/master/libcontainer/configs/device_defaults.go
      //
      // YARN by default runs docker container inside cgroup, if we setup cgroups
      // devices.deny for the parent cgroup for launched container, we can see
      // errors like: failed to write c *:* m to devices.allow:
      // write path-to-parent-cgroup/<container-id>/devices.allow:
      // operation not permitted.
      //
      // To avoid this happen, if docker is requested when container being
      // launched, we will not setup devices.deny for the container. Instead YARN
      // will pass --device parameter to docker engine. See NvidiaDockerV1CommandPlugin
      try {
        // Execute c-e to setup GPU isolation before launch the container
        PrivilegedOperation privilegedOperation = new PrivilegedOperation(
            PrivilegedOperation.OperationType.GPU,
            Arrays.asList(CONTAINER_ID_CLI_OPTION, containerIdStr));
        if (!allocation.getDeniedGPUs().isEmpty()) {
          List<Integer> minorNumbers = new ArrayList<>();
          for (GpuDevice deniedGpu : allocation.getDeniedGPUs()) {
            minorNumbers.add(deniedGpu.getMinorNumber());
          }
          privilegedOperation.appendArgs(Arrays.asList(EXCLUDED_GPUS_CLI_OPTION,
              StringUtils.join(",", minorNumbers)));
        }

        privilegedOperationExecutor.executePrivilegedOperation(
            privilegedOperation, true);
      } catch (PrivilegedOperationException e) {
        cGroupsHandler.deleteCGroup(CGroupsHandler.CGroupController.DEVICES,
            containerIdStr);
        LOG.warn("Could not update cgroup for container", e);
        throw new ResourceHandlerException(e);
      }

      List<PrivilegedOperation> ret = new ArrayList<>();
      ret.add(new PrivilegedOperation(
          PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP,
          PrivilegedOperation.CGROUP_ARG_PREFIX + cGroupsHandler
              .getPathForCGroupTasks(CGroupsHandler.CGroupController.DEVICES,
                  containerIdStr)));

      return ret;
    }
    return null;
  }
@Test
  public void testAssignedGpuWillBeCleanedUpWhenStoreOpFails()
      throws Exception {
    initializeGpus();

    doThrow(new IOException("Exception ...")).when(mockNMStateStore)
        .storeAssignedResources(
        any(Container.class), anyString(), anyList());

    boolean exception = false;
    /* Start container 1, asks 3 containers */
    try {
      gpuResourceHandler.preStart(mockContainerWithGpuRequest(1,
          createResourceRequest(3)));
    } catch (ResourceHandlerException e) {
      exception = true;
    }

    assertTrue("preStart should throw exception", exception);

    // After preStart, we still have 4 available GPU since the store op failed.
    verifyNumberOfAvailableGpus(4, gpuResourceHandler);
  }