Skip to content

Unable to Train Small Version on Cityscapes #2

Open
Open
Unable to Train Small Version on Cityscapes#2

Description

@Michel-Liao
Michel-Liao
opened on Jul 10, 2024

Execution Command

Running python -m torch.distributed.launch --nproc_per_node=4 tools/train.py --config-file configs/cityscapes/pvt_small_FPT.yaml on four GTX 1080s.

Error

Found an error within /segmentron/data/dataloader/cityscapes.py. In _get_city_pairs(), the code doesn't account for if split='test'. Bug partially fixed if you change if split in ('train', 'val'): in _get_city_pairs() to if split in ('train', 'val', 'test'):.

However, when running the execution command again, I get an AssertionError: AssertionError== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]

Error Log

python -m torch.distributed.launch --nproc_per_node=4 tools/train.py --config-file configs/cityscapes/pvt_small_FPT.yaml
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
apex is not installed
apex is not installed
apex is not installedapex is not installed

INFO:Segmentron:Using 4 GPUs
2024-07-10 15:09:21,711 Segmentron INFO: Using 4 GPUs
INFO:Segmentron:Namespace(config_file='configs/cityscapes/pvt_small_FPT.yaml', device='cuda', distributed=True, input_img='tools/demo_vis.png', local_rank=0, log_iter=10, no_cuda=False, num_gpus=4, opts=[], resume=None, skip_val=False, test=False, val_epoch=1, vis=False)
2024-07-10 15:09:21,711 Segmentron INFO: Namespace(config_file='configs/cityscapes/pvt_small_FPT.yaml', device='cuda', distributed=True, input_img='tools/demo_vis.png', local_rank=0, log_iter=10, no_cuda=False, num_gpus=4, opts=[], resume=None, skip_val=False, test=False, val_epoch=1, vis=False)
INFO:Segmentron:{
        "SEED": 1024,
        "TIME_STAMP": "2024-07-10-15-08",
        "ROOT_PATH": "/n/fs/pvl-dpvorot/t4t_internal",
        "PHASE": "train",
        "DATASET": {
                "NAME": "cityscape",
                "MEAN": [
                        0.485,
                        0.456,
                        0.406
                ],
                "STD": [
                        0.229,
                        0.224,
                        0.225
                ],
                "IGNORE_INDEX": -1,
                "WORKERS": 8,
                "MODE": "testval"
        },
        "AUG": {
                "MIRROR": true,
                "BLUR_PROB": 0.0,
                "BLUR_RADIUS": 0.0,
                "COLOR_JITTER": null,
                "CROP": true
        },
        "TRAIN": {
                "EPOCHS": 200,
                "ITERS": 40000,
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "BASE_SIZE": 512,
                "MODEL_SAVE_DIR": "workdirs/cityscapes/pvt_small_FPT",
                "LOG_SAVE_DIR": "workdirs/",
                "PRETRAINED_MODEL_PATH": "",
                "BACKBONE_PRETRAINED": true,
                "BACKBONE_PRETRAINED_PATH": "pretrained/pvt_small.pth",
                "RESUME_MODEL_PATH": "",
                "SYNC_BATCH_NORM": true,
                "SNAPSHOT_EPOCH": 1,
                "APEX": true
        },
        "SOLVER": {
                "LR": 5e-05,
                "OPTIMIZER": "adamw",
                "EPSILON": 1e-08,
                "MOMENTUM": 0.9,
                "WEIGHT_DECAY": 0.0001,
                "DECODER_LR_FACTOR": 10.0,
                "LR_SCHEDULER": "poly",
                "POLY": {
                        "POWER": 0.9
                },
                "STEP": {
                        "GAMMA": 0.1,
                        "DECAY_EPOCH": [
                                10,
                                20
                        ]
                },
                "WARMUP": {
                        "EPOCHS": 0.0,
                        "FACTOR": 0.3333333333333333,
                        "METHOD": "linear"
                },
                "OHEM": false,
                "AUX": false,
                "AUX_WEIGHT": 0.4,
                "LOSS_NAME": ""
        },
        "TEST": {
                "TEST_MODEL_PATH": "workdirs/cityscapes/pvt_small_FPT/best_model.pth",
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "SCALES": [
                        1.0
                ],
                "FLIP": false
        },
        "VISUAL": {
                "OUTPUT_DIR": "../runs/visual/"
        },
        "MODEL": {
                "MODEL_NAME": "PVT_FPT",
                "BACKBONE": "pvt_small",
                "BACKBONE_SCALE": 1.0,
                "MULTI_LOSS_WEIGHT": [
                        1.0
                ],
                "DEFAULT_GROUP_NUMBER": 32,
                "DEFAULT_EPSILON": 1e-05,
                "BN_TYPE": "BN",
                "BN_EPS_FOR_ENCODER": null,
                "BN_EPS_FOR_DECODER": null,
                "OUTPUT_STRIDE": 16,
                "BN_MOMENTUM": null,
                "EMB_CHANNELS": 128,
                "DANET": {
                        "MULTI_DILATION": null,
                        "MULTI_GRID": false
                },
                "DEEPLABV3_PLUS": {
                        "USE_ASPP": true,
                        "ENABLE_DECODER": true,
                        "ASPP_WITH_SEP_CONV": true,
                        "DECODER_USE_SEP_CONV": true
                },
                "OCNet": {
                        "OC_ARCH": "base"
                },
                "ENCNET": {
                        "SE_LOSS": true,
                        "SE_WEIGHT": 0.2,
                        "LATERAL": true
                },
                "CCNET": {
                        "RECURRENCE": 2
                },
                "CGNET": {
                        "STAGE2_BLOCK_NUM": 3,
                        "STAGE3_BLOCK_NUM": 21
                },
                "POINTREND": {
                        "BASEMODEL": "DeepLabV3_Plus"
                },
                "HRNET": {
                        "PRETRAINED_LAYERS": [
                                "*"
                        ],
                        "STEM_INPLANES": 64,
                        "FINAL_CONV_KERNEL": 1,
                        "WITH_HEAD": true,
                        "STAGE1": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 1,
                                "NUM_BLOCKS": [
                                        1
                                ],
                                "NUM_CHANNELS": [
                                        32
                                ],
                                "BLOCK": "BOTTLENECK",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE2": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 2,
                                "NUM_BLOCKS": [
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE3": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 3,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE4": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 4,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128,
                                        256
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        }
                },
                "TRANSLAB": {
                        "BOUNDARY_WEIGHT": 5
                },
                "TRANS4TRANS": {
                        "embed_dim": 256,
                        "depth": 4,
                        "num_heads": 8,
                        "mlp_ratio": 3.0,
                        "hid_dim": 64
                }
        }
}
2024-07-10 15:09:21,712 Segmentron INFO: {
        "SEED": 1024,
        "TIME_STAMP": "2024-07-10-15-08",
        "ROOT_PATH": "/n/fs/pvl-dpvorot/t4t_internal",
        "PHASE": "train",
        "DATASET": {
                "NAME": "cityscape",
                "MEAN": [
                        0.485,
                        0.456,
                        0.406
                ],
                "STD": [
                        0.229,
                        0.224,
                        0.225
                ],
                "IGNORE_INDEX": -1,
                "WORKERS": 8,
                "MODE": "testval"
        },
        "AUG": {
                "MIRROR": true,
                "BLUR_PROB": 0.0,
                "BLUR_RADIUS": 0.0,
                "COLOR_JITTER": null,
                "CROP": true
        },
        "TRAIN": {
                "EPOCHS": 200,
                "ITERS": 40000,
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "BASE_SIZE": 512,
                "MODEL_SAVE_DIR": "workdirs/cityscapes/pvt_small_FPT",
                "LOG_SAVE_DIR": "workdirs/",
                "PRETRAINED_MODEL_PATH": "",
                "BACKBONE_PRETRAINED": true,
                "BACKBONE_PRETRAINED_PATH": "pretrained/pvt_small.pth",
                "RESUME_MODEL_PATH": "",
                "SYNC_BATCH_NORM": true,
                "SNAPSHOT_EPOCH": 1,
                "APEX": true
        },
        "SOLVER": {
                "LR": 5e-05,
                "OPTIMIZER": "adamw",
                "EPSILON": 1e-08,
                "MOMENTUM": 0.9,
                "WEIGHT_DECAY": 0.0001,
                "DECODER_LR_FACTOR": 10.0,
                "LR_SCHEDULER": "poly",
                "POLY": {
                        "POWER": 0.9
                },
                "STEP": {
                        "GAMMA": 0.1,
                        "DECAY_EPOCH": [
                                10,
                                20
                        ]
                },
                "WARMUP": {
                        "EPOCHS": 0.0,
                        "FACTOR": 0.3333333333333333,
                        "METHOD": "linear"
                },
                "OHEM": false,
                "AUX": false,
                "AUX_WEIGHT": 0.4,
                "LOSS_NAME": ""
        },
        "TEST": {
                "TEST_MODEL_PATH": "workdirs/cityscapes/pvt_small_FPT/best_model.pth",
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "SCALES": [
                        1.0
                ],
                "FLIP": false
        },
        "VISUAL": {
                "OUTPUT_DIR": "../runs/visual/"
        },
        "MODEL": {
                "MODEL_NAME": "PVT_FPT",
                "BACKBONE": "pvt_small",
                "BACKBONE_SCALE": 1.0,
                "MULTI_LOSS_WEIGHT": [
                        1.0
                ],
                "DEFAULT_GROUP_NUMBER": 32,
                "DEFAULT_EPSILON": 1e-05,
                "BN_TYPE": "BN",
                "BN_EPS_FOR_ENCODER": null,
                "BN_EPS_FOR_DECODER": null,
                "OUTPUT_STRIDE": 16,
                "BN_MOMENTUM": null,
                "EMB_CHANNELS": 128,
                "DANET": {
                        "MULTI_DILATION": null,
                        "MULTI_GRID": false
                },
                "DEEPLABV3_PLUS": {
                        "USE_ASPP": true,
                        "ENABLE_DECODER": true,
                        "ASPP_WITH_SEP_CONV": true,
                        "DECODER_USE_SEP_CONV": true
                },
                "OCNet": {
                        "OC_ARCH": "base"
                },
                "ENCNET": {
                        "SE_LOSS": true,
                        "SE_WEIGHT": 0.2,
                        "LATERAL": true
                },
                "CCNET": {
                        "RECURRENCE": 2
                },
                "CGNET": {
                        "STAGE2_BLOCK_NUM": 3,
                        "STAGE3_BLOCK_NUM": 21
                },
                "POINTREND": {
                        "BASEMODEL": "DeepLabV3_Plus"
                },
                "HRNET": {
                        "PRETRAINED_LAYERS": [
                                "*"
                        ],
                        "STEM_INPLANES": 64,
                        "FINAL_CONV_KERNEL": 1,
                        "WITH_HEAD": true,
                        "STAGE1": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 1,
                                "NUM_BLOCKS": [
                                        1
                                ],
                                "NUM_CHANNELS": [
                                        32
                                ],
                                "BLOCK": "BOTTLENECK",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE2": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 2,
                                "NUM_BLOCKS": [
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE3": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 3,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE4": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 4,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128,
                                        256
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        }
                },
                "TRANSLAB": {
                        "BOUNDARY_WEIGHT": 5
                },
                "TRANS4TRANS": {
                        "embed_dim": 256,
                        "depth": 4,
                        "num_heads": 8,
                        "mlp_ratio": 3.0,
                        "hid_dim": 64
                }
        }
}
INFO:Segmentron:Found 2975 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/train
2024-07-10 15:09:23,330 Segmentron INFO: Found 2975 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/train
INFO:Segmentron:Found 500 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/val
2024-07-10 15:09:23,606 Segmentron INFO: Found 500 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/val
INFO:Segmentron:Found 1525 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/test
2024-07-10 15:09:24,414 Segmentron INFO: Found 1525 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/test
INFO:Segmentron:Load backbone pretrained model from pretrained/pvt_small.pth
2024-07-10 15:09:27,335 Segmentron INFO: Load backbone pretrained model from pretrained/pvt_small.pth
INFO:Segmentron:_IncompatibleKeys(missing_keys=[], unexpected_keys=['cls_token', 'norm.weight', 'norm.bias', 'head.weight', 'head.bias'])
2024-07-10 15:09:35,993 Segmentron INFO: _IncompatibleKeys(missing_keys=[], unexpected_keys=['cls_token', 'norm.weight', 'norm.bias', 'head.weight', 'head.bias'])
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
        model = MODEL_REGISTRY.get(model_name)()model = MODEL_REGISTRY.get(model_name)()

          File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
model = MODEL_REGISTRY.get(model_name)()model = MODEL_REGISTRY.get(model_name)()

  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
    == cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]
    AssertionError== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]    

== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]    
== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]
AssertionError
AssertionError
AssertionError

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions