diff --git a/README.md b/README.md index cb36813..deda275 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Container Explorer provides the following functionalities: - Exploring images - Exploring snapshots - Exploring contents +- Exploring container drift - Mounting containers - Support JSON output @@ -28,39 +29,43 @@ The figure below shows the output of the container-explorer --help command. ```text NAME: container-explorer - A standalone utility to explore container details - + USAGE: - container-explorer [global options] command [command options] [arguments...] - + ce [global options] command [command options] [arguments...] + VERSION: - 0.0.2 - + 0.2.1 (20240502) + DESCRIPTION: A standalone utility to explore container details. - + Container explorer supports exploring containers managed using containerd and - docker. The utility also supports exploring containers created and managed - using Kubernetes. - - + docker. The utility also supports exploring containers created and managed using + Kubernetes. + + COMMANDS: list, ls Lists container related information info show internal information mount mount a container to a mount point mount-all, mount_all mount all containers + drift, diff identifies container filesystem changes help, h Shows a list of commands or help for one command - + GLOBAL OPTIONS: --debug enable debug messages --containerd-root value, -c value specify containerd root directory --image-root value, -i value specify mount point for a disk image --metadata-file value, -m value specify the path to containerd metadata file i.e. meta.db --snapshot-metadata-file value, -s value specify the path to containerd snapshot metadata file i.e. metadata.db. + --use-layer-cache attempt to use cached layers where layers are symlinks + --layer-cache value cached layer folder within the snapshot root (default: "layers") --namespace value, -n value specify container namespace (default: "default") --docker-managed specify docker manages standalone or Kubernetes containers --docker-root value specify docker root directory. This is only used with flag --docker-managed --support-container-data value a yaml file containing information about support containers --output value output format in json, table. Default is table (default: "table") + --output-file value, -o value output file to save the content --help, -h show help --version, -v print the version ``` @@ -146,6 +151,18 @@ forensic VM as `/dev/sdb`. drwxr-xr-x 1 root root 4096 Feb 5 08:54 f3c910583a81e7441e2cbd209b72afa4740e676ff8d82f2c74fdc5c78e179c10 ``` +6. See filesystem changes + + ```shell + sudo ce -i /mnt/case/ --output json --support-container-data supportcontainer.yaml drift + ``` + + In order to see drift of a particular container, supply the container ID with drift/diff + + ```shell + sudo ce -i /mnt/case/ --output json --support-container-data supportcontainer.yaml drift f3c910583a81e7441e2cbd209b72afa4740e676ff8d82f2c74fdc5c78e179c10 + ``` + 6. Use your favorite forensic tool to process mounted containers. ## Mounting Disk Image diff --git a/cmd/commands/drift.go b/cmd/commands/drift.go new file mode 100644 index 0000000..92d54d0 --- /dev/null +++ b/cmd/commands/drift.go @@ -0,0 +1,116 @@ +/* +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package commands + +import ( + "fmt" + "runtime" + "os" + "strings" + "text/tabwriter" + + log "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var DriftCommand = cli.Command{ + Name: "drift", + Aliases: []string{"diff"}, + Usage: "identifies container filesystem changes", + Description: "identifies container filesystem changes for all containers", + ArgsUsage: "[containerID]", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "filter", + Usage: "comma separated label filter using key=value pair", + }, + cli.BoolFlag{ + Name: "mount-support-containers", + Usage: "mount Kubernetes supporting containers", + }, + }, + Action: func(clictx *cli.Context) error { + // Mounting a container is only supported on a Linux operating system. + if runtime.GOOS != "linux" { + return fmt.Errorf("feature is only supported on Linux") + } + output := clictx.GlobalString("output") + outputfile := clictx.GlobalString("output-file") + filter := clictx.String("filter") + + // Getting container ID positional arg + var containerID string + if clictx.Args().Present() { + containerID = clictx.Args().First() + } + + ctx, exp, cancel, err := explorerEnvironment(clictx) + if err != nil { + return err + } + defer cancel() + + drifts, err := exp.ContainerDrift(ctx, filter, !clictx.Bool("mount-support-containers"), containerID) + if err != nil { + log.WithField("message", err).Error("retrieving container drift") + if output == "json" && outputfile != "" { + data := []string{} + writeOutputFile(data, outputfile) + } + return nil + } + // Handle output formats + if strings.ToLower(output) == "json" { + if outputfile != "" { + writeOutputFile(drifts, outputfile) + } else { + printAsJSON(drifts) + } + return nil + } + + // Default to table output + tw := tabwriter.NewWriter(os.Stdout, 1, 8, 1, '\t', 0) + defer tw.Flush() + + if output == "table" { + // Define the header + fmt.Fprintf(tw, "CONTAINER ID\tADDED/MODIFIED\tDELETED\n") + } + + for _, drift := range drifts { + switch strings.ToLower(output) { + case "json_line": + printAsJSONLine(drift) + default: + // Prepare the data for display + addedOrModified := strings.Join(drift.AddedOrModified, ", ") + inaccessibleFiles := strings.Join(drift.InaccessibleFiles, ", ") + + displayValues := fmt.Sprintf("%s\t%s\t%s", + drift.ContainerID, + addedOrModified, + inaccessibleFiles, + ) + + fmt.Fprintf(tw, "%v\n", displayValues) + } + } + // default + return nil + }, +} diff --git a/cmd/main.go b/cmd/main.go index 7310911..ffdf177 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -126,6 +126,7 @@ func main() { cecommands.InfoCommand, cecommands.MountCommand, cecommands.MountAllCommand, + cecommands.DriftCommand, } app.Before = func(context *cli.Context) error { diff --git a/explorers/container.go b/explorers/container.go index 600001d..1df955f 100644 --- a/explorers/container.go +++ b/explorers/container.go @@ -35,3 +35,9 @@ type Container struct { Running bool ExposedPorts []string } + +type Drift struct { + ContainerID string + AddedOrModified []string + InaccessibleFiles []string +} diff --git a/explorers/containerd/containerd.go b/explorers/containerd/containerd.go index 07d9e32..43a4892 100644 --- a/explorers/containerd/containerd.go +++ b/explorers/containerd/containerd.go @@ -25,6 +25,7 @@ import ( "os/exec" "path/filepath" "strings" + "syscall" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/metadata" @@ -615,6 +616,200 @@ func (e *explorer) MountAllContainers(ctx context.Context, mountpoint string, fi return nil } +// ScanDiffDirectory identifies added or modified files in the diff directory +func ScanDiffDirectory(diffDir string) (addedOrModified []string, inaccessibleFiles []string, err error) { + err = filepath.Walk(diffDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + relativePath, relErr := filepath.Rel(diffDir, path) + if relErr != nil { + return relErr + } + inaccessibleFiles = append(inaccessibleFiles, relativePath) + return nil // Continue walking despite the error + } + if !info.IsDir() { + relativePath, relErr := filepath.Rel(diffDir, path) + if relErr != nil { + return relErr + } + + // Check if the file is a whiteout files + if info.Mode()&os.ModeCharDevice != 0 { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + rdev := stat.Rdev + + // Extract major and minor device numbers + major := (rdev >> 8) & 0xfff + minor := (rdev & 0xff) | ((rdev >> 12) & 0xfff00) + + if major == 0 && minor == 0 { + // This is a whiteout file + inaccessibleFiles = append(inaccessibleFiles, relativePath) + return nil + } + } + } + + // Check if the file is not a symbolic link + if info.Mode()&os.ModeSymlink == 0 { + // Check if the file has executable permissions + mode := info.Mode().Perm() + if mode&0111 != 0 { + // The file is executable by owner, group, or others + relativePath += " (executable)" + } + } + + addedOrModified = append(addedOrModified, relativePath) + } + return nil + }) + return +} + +// ContainerDrift finds drifted files from all the containers +func (e *explorer) ContainerDrift(ctx context.Context, filter string, skipsupportcontainers bool, containerID string) ([]explorers.Drift, error) { + var drifts []explorers.Drift + ctrs, err := e.ListContainers(ctx) + if err != nil { + return nil, err + } + + filters := strings.Split(filter, ",") + + for _, ctr := range ctrs { + // If containerID is supplied & doesn't match skip + if containerID != "" && ctr.ID != containerID { + continue + } + + // Skip Kubernetes suppot containers + if skipsupportcontainers && ctr.SupportContainer { + log.WithFields(log.Fields{ + "namespace": ctr.Namespace, + "containerid": ctr.ID, + }).Info("skip mounting Kubernetes containers") + + continue + } + + // Only analyze containers matching the filter. + analyze := true + for _, f := range filters { + if !strings.Contains(f, "=") { + continue + } + + key := strings.Split(f, "=")[0] + value := strings.Split(f, "=")[1] + + labelValue, ok := ctr.Labels[key] + if !ok { + analyze = false + break + } + + if labelValue != value { + analyze = false + break + } + } + + if !analyze { + continue + } + + e.snapshot = "" + ctx = namespaces.WithNamespace(ctx, ctr.Namespace) + store := metadata.NewContainerStore(metadata.NewDB(e.mdb, nil, nil)) + + container, err := store.Get(ctx, ctr.ID) + if err != nil { + return nil, fmt.Errorf("failed getting container information %v", err) + } + // Snapshot database metadata.db access + opts := bolt.Options{ + ReadOnly: true, + } + if e.snapshot == "" { + snapshotterFolder := e.SnapshotRoot(container.Snapshotter) + if snapshotterFolder != "unknown" { + e.snapshot = filepath.Join(snapshotterFolder, "metadata.db") + } + } + log.WithFields(log.Fields{ + "snapshotter": container.Snapshotter, + "snapshotKey": container.SnapshotKey, + "image": container.Image, + "snapshotterFolder": e.snapshot, + }).Debug("container snapshotter") + ssdb, err := bolt.Open(e.snapshot, 0444, &opts) + if err != nil { + return nil, fmt.Errorf("failed to open snapshot database %v", err) + } + // snapshot store + ssstore := NewSnaptshotStore(e.root, e.layercache, e.mdb, ssdb) + hasWorkDir := false + snapshotRoot, _ := filepath.Split(e.snapshot) + matches, _ := filepath.Glob(filepath.Join(snapshotRoot, "snapshots/*/work")) + if len(matches) > 0 { + hasWorkDir = true + } + if container.Snapshotter == "native" { + upperdir, err := ssstore.NativePath(ctx, container) + log.WithFields(log.Fields{ + "upperdir": upperdir, + }).Debug("native directories") + if err != nil { + return nil, fmt.Errorf("failed to get native path %v", err) + } + } else if hasWorkDir { + lowerdir, upperdir, workdir, err := ssstore.OverlayPath(ctx, container) + log.WithFields(log.Fields{ + "lowerdir": lowerdir, + "upperdir": upperdir, + "workdir": workdir, + }).Debug("overlay directories") + + log.WithFields(log.Fields{ + "container ID": ctr.ID, + }).Debug("Checking drift for container") + if err != nil { + return nil, fmt.Errorf("failed to get overlay path %v", err) + } + if lowerdir == "" { + return nil, fmt.Errorf("lowerdir is empty") + } + + // Scan upperdir + addedOrModified, inaccessibleFiles, err := ScanDiffDirectory(upperdir) + if err != nil { + return nil, fmt.Errorf("failed to scan diff directory: %v", err) + } + + drift := explorers.Drift{ContainerID: ctr.ID, AddedOrModified: addedOrModified, InaccessibleFiles: inaccessibleFiles} + + drifts = append(drifts, drift) + + for _, path := range addedOrModified { + log.WithFields(log.Fields{ + "A ": path}).Debug("added or modified files") + } + if len(inaccessibleFiles) > 0 { + for _, path := range inaccessibleFiles { + log.WithFields(log.Fields{ + "D ": path}).Debug("deleted files") + } + } + } else { + log.Error("Unsupported snapshotter ", container.Snapshotter) + } + } + + // default + return drifts, nil +} + // Close releases the internal resources func (e *explorer) Close() error { return e.mdb.Close() @@ -680,3 +875,4 @@ func imageBasename(image string) string { } return imagebase } + diff --git a/explorers/docker/docker.go b/explorers/docker/docker.go index 94bd258..81f8e0d 100644 --- a/explorers/docker/docker.go +++ b/explorers/docker/docker.go @@ -26,6 +26,7 @@ import ( "path/filepath" "strings" "time" + "syscall" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/images" @@ -455,6 +456,201 @@ func (e *explorer) MountAllContainers(ctx context.Context, mountpoint string, fi return nil } +// ScanDiffDirectory identifies added or modified files in the diff directory +func ScanDiffDirectory(diffDir string) (addedOrModified []string, inaccessibleFiles []string, err error) { + err = filepath.Walk(diffDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + relativePath, relErr := filepath.Rel(diffDir, path) + if relErr != nil { + return relErr + } + inaccessibleFiles = append(inaccessibleFiles, relativePath) + return nil // Continue walking despite the error + } + if !info.IsDir() { + relativePath, relErr := filepath.Rel(diffDir, path) + if relErr != nil { + return relErr + } + + // Check if the file is a whiteout files + if info.Mode()&os.ModeCharDevice != 0 { + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + rdev := stat.Rdev + + // Extract major and minor device numbers + major := (rdev >> 8) & 0xfff + minor := (rdev & 0xff) | ((rdev >> 12) & 0xfff00) + + if major == 0 && minor == 0 { + // Whiteout file + inaccessibleFiles = append(inaccessibleFiles, relativePath) + return nil + } + } + } + + // Check if the file is not a symbolic link + if info.Mode()&os.ModeSymlink == 0 { + // Check if the file has executable permissions + mode := info.Mode().Perm() + if mode&0111 != 0 { + // The file is executable by owner, group, or others + relativePath += " (executable)" + } + } + + addedOrModified = append(addedOrModified, relativePath) + } + return nil + }) + return +} + +// ContainerDrift finds drifted files from all the containers +func (e *explorer) ContainerDrift(ctx context.Context, filter string, skipsupportcontainers bool, containerID string) ([]explorers.Drift, error) { + var drifts []explorers.Drift + containersdir := filepath.Join(e.root, containersDirName) + log.WithField("containersdir", containersdir).Debug("docker containers directory") + + containerids, err := e.GetContainerIDs(ctx, containersdir) + if err != nil { + return nil, fmt.Errorf("failed listing containers ID %v", err) + } + if containerids == nil { + return nil, fmt.Errorf("no container ID returned") + } + + filters := strings.Split(filter, ",") + + for _, containerid := range containerids { + cecontainer, err := e.GetCEContainer(ctx, containerid) + if err != nil { + log.WithField("containerid", containerid).Error("getting container details") + log.WithField("containerid", containerid).Warn("skipping container mount") + continue + } + + // If containerID is supplied & doesn't match skip + if containerID != "" && cecontainer.ID != containerID { + continue + } + + if skipsupportcontainers && cecontainer.SupportContainer { + log.WithFields(log.Fields{ + "namespace": cecontainer.Namespace, + "containerid": cecontainer.ID, + }).Info("skip mounting Kubernetes support container") + continue + } + + // Only analyze containers matching the filter. + analyze := true + for _, f := range filters { + if !strings.Contains(f, "=") { + continue + } + + key := strings.Split(f, "=")[0] + value := strings.Split(f, "=")[1] + + labelValue, ok := cecontainer.Labels[key] + if !ok { + analyze = false + break + } + + if labelValue != value { + analyze = false + break + } + } + + if !analyze { + continue + } + + container, err := e.GetContainer(ctx, cecontainer.ID) + if err != nil { + return nil, fmt.Errorf("getting container %v", err) + } + + containerMountIDPath := filepath.Join(e.root, repositoriesDirName, container.Driver, "layerdb", "mounts", cecontainer.ID, "mount-id") + log.WithField("containerMountIDPath", containerMountIDPath).Debug("container mount-id path") + + mountIDByte, err := ioutil.ReadFile(containerMountIDPath) + if err != nil { + return nil, fmt.Errorf("reading container mount-id") + } + + mountID := string(mountIDByte) + log.WithField("mount-id", mountID).Debug("container mount-id") + + // build container lower directory + lowerdirpath := filepath.Join(e.root, container.Driver, mountID, lowerdirName) + log.WithField("lowerdirpath", lowerdirpath).Debug("container lowerdir path") + data, err := ioutil.ReadFile(lowerdirpath) + if err != nil { + return nil, fmt.Errorf("reading lower file %v", err) + } + + var lowerdir string + for i, ldir := range strings.Split(string(data), ":") { + ldirpath := filepath.Join(e.root, container.Driver, ldir) + if i == 0 { + lowerdir = ldirpath + continue + } + lowerdir = fmt.Sprintf("%s:%s", lowerdir, ldirpath) + } + + upperdir := filepath.Join(e.root, container.Driver, mountID, "diff") + workdir := filepath.Join(e.root, container.Driver, mountID, "work") + + log.WithFields(log.Fields{ + "lowerdir": lowerdir, + "upperdir": upperdir, + "workdir": workdir, + }).Debug("container overlay directories") + + log.WithFields(log.Fields{ + "container ID": cecontainer.ID, + }).Debug("checking drift for container") + if err != nil { + return nil, fmt.Errorf("failed to get overlay path %v", err) + } + + if lowerdir == "" { + return nil, fmt.Errorf("lowerdir is empty") + } + + // Scan upperdir + addedOrModified, inaccessibleFiles, err := ScanDiffDirectory(upperdir) + if err != nil { + return nil, fmt.Errorf("failed to scan diff directory: %v", err) + } + drift := explorers.Drift{ + ContainerID: cecontainer.ID, + AddedOrModified: addedOrModified, + InaccessibleFiles: inaccessibleFiles, + } + + drifts = append(drifts, drift) + for _, path := range addedOrModified { + log.WithFields(log.Fields{ + "A ": path}).Debug("added or modified files") + } + if len(inaccessibleFiles) > 0 { + for _, path := range inaccessibleFiles { + log.WithFields(log.Fields{ + "D ": path}).Debug("deleted files") + } + } + } + // default + return drifts, nil +} + // Close releases internal resources. func (e *explorer) Close() error { return e.mdb.Close() diff --git a/explorers/explorers.go b/explorers/explorers.go index fd06d7e..a850758 100644 --- a/explorers/explorers.go +++ b/explorers/explorers.go @@ -60,6 +60,9 @@ type ContainerExplorer interface { // MountAllContainer mounts all containers to the specfied path MountAllContainers(ctx context.Context, mountpoint string, filter string, skipsupportcontainers bool) error + // ContainerDrift identifies container filesystem changes + ContainerDrift(ctx context.Context, filter string, skipsupportcontainers bool, containerID string) ([]Drift, error) + // Close releases the internal resources Close() error }