feat(eval): update how to submit results on the 2rd leaderboard version.

Author: Kin-Zhang

2_eval.py

@@ -29,9 +29,10 @@ def main(cfg):
         sys.exit(1)
 
     checkpoint_params = DictConfig(torch.load(cfg.checkpoint)["hyper_parameters"])
-    cfg.output = checkpoint_params.cfg.output + f"-{cfg.av2_mode}"
+    cfg.output = checkpoint_params.cfg.output + f"-{cfg.av2_mode}-v{cfg.leaderboard_version}"
     cfg.model.update(checkpoint_params.cfg.model)
     mymodel = ModelWrapper.load_from_checkpoint(cfg.checkpoint, cfg=cfg, eval=True)
+    print(f"\n---LOG[eval]: Loaded model from {cfg.checkpoint}. The model is {checkpoint_params.cfg.model.name}.\n")
 
     wandb_logger = WandbLogger(save_dir=output_dir,
                                entity="kth-rpl",
@@ -42,7 +43,7 @@ def main(cfg):
     trainer = pl.Trainer(logger=wandb_logger, devices=1)
     # NOTE(Qingwen): search & check: def eval_only_step_(self, batch, res_dict)
     trainer.validate(model = mymodel, \
-                     dataloaders = DataLoader(HDF5Dataset(cfg.dataset_path + f"/{cfg.av2_mode}", eval=True), batch_size=1, shuffle=False))
+                     dataloaders = DataLoader(HDF5Dataset(cfg.dataset_path + f"/{cfg.av2_mode}", eval=True, leaderboard_version=cfg.leaderboard_version), batch_size=1, shuffle=False))
     wandb.finish()
 
 if __name__ == "__main__":

README.md

@@ -7,6 +7,9 @@ DeFlow: Decoder of Scene Flow Network in Autonomous Driving
 [![video](https://img.shields.io/badge/video-YouTube-FF0000?logo=youtube&logoColor=white)](https://youtu.be/bZ4uUv0nDa0)
 
 Task: Scene Flow Estimation in Autonomous Driving. 
+
+🔥 2024/07/02: Check the self-supervised version in our new ECCV'24 [SeFlow](https://github.com/KTH-RPL/SeFlow). The 1st ranking in new leaderboard among self-supervise methods.
+
 Pre-trained weights for models are available in [Zenodo](https://zenodo.org/records/12173874) or [Onedrive link](https://hkustconnect-my.sharepoint.com/:f:/g/personal/qzhangcb_connect_ust_hk/Et85xv7IGMRKgqrVeJEVkMoB_vxlcXk6OZUyiPjd4AArIg?e=lqRGhx). 
 Check usage in [2. Evaluation](#2-evaluation) or [3. Visualization](#3-visualization). 
 
@@ -72,10 +75,6 @@ Benchmarking and baseline methods:
 ```bash
 python 1_train.py model=fastflow3d lr=2e-6 epochs=50 batch_size=16
 python 1_train.py model=deflow lr=2e-6 epochs=50 batch_size=16
-
-# for nsfp no need train but optimize iteration running
-python 2_eval.py model=nsfp 
-python 2_eval.py model=fast_nsfp
 ```
 
 To help community benchmarking, we provide our weights including fastflow3d, deflow [Onedrive link](https://hkustconnect-my.sharepoint.com/:f:/g/personal/qzhangcb_connect_ust_hk/Et85xv7IGMRKgqrVeJEVkMoB_vxlcXk6OZUyiPjd4AArIg?e=lqRGhx). These checkpoints also include parameters and status of that epoch inside it. If you are interested in weights of ablation studies, please contact us.
@@ -92,23 +91,28 @@ Since in training, we save all hyper-parameters and model checkpoints, the only
 wget https://zenodo.org/records/12173874/files/deflow_best.ckpt
 
 python 2_eval.py checkpoint=/home/kin/deflow_best.ckpt av2_mode=val # it will directly prints all metric
-python 2_eval.py checkpoint=/home/kin/deflow_best.ckpt av2_mode=test # it will output the av2_submit.zip for you to submit to leaderboard
+# it will output the av2_submit.zip or av2_submit_v2.zip for you to submit to leaderboard
+python 2_eval.py checkpoint=/home/kin/deflow_best.ckpt av2_mode=test leaderboard_version=1
+python 2_eval.py checkpoint=/home/kin/deflow_best.ckpt av2_mode=test leaderboard_version=2
 ```
 
 Check all detailed result files (presented in our paper Table 1) in [this discussion](https://github.com/KTH-RPL/DeFlow/discussions/2).
 
-To submit to the Online Leaderboard, the last step will tell you the resulting path, copy it here:
+To submit to the Online Leaderboard, if you select `av2_mode=test`, it should be a zip file for you to submit to the leaderboard.
+Note: The leaderboard result in DeFlow main paper is [version 1](https://eval.ai/web/challenges/challenge-page/2010/evaluation), as [version 2](https://eval.ai/web/challenges/challenge-page/2210/overview) is updated after DeFlow paper.
+
 ```bash
-# you will find there is a av2_submit.zip in the folder now. since the env is different and conflict we set new one:
+# since the env may conflict we set new on deflow, we directly create new one:
 mamba create -n py37 python=3.7
 mamba activate py37
 pip install "evalai"
 
 # Step 2: login in eval and register your team
-evalai set_token <your token>
+evalai set-token <your token>
 
 # Step 3: Submit to leaderboard
 evalai challenge 2010 phase 4018 submit --file av2_submit.zip --large --private
+evalai challenge 2210 phase 4396 submit --file av2_submit_v2.zip --large --private
 ```
 
 ## 3. Visualization

assets/docs/index_eval_v2.pkl

@@ -4,6 +4,9 @@ checkpoint: /home/kin/model_zoo/deflow.ckpt
 av2_mode: val # [val, test]
 save_res: False # [True, False]
 
+leaderboard_version: 1 # [1, 2]
+supervised_flag: True # [True, False], whether you use any label from the dataset
+
 # no need to change
 slurm_id: 00000
 output: ${model.name}-${slurm_id}

conf/eval.yaml

@@ -43,7 +43,7 @@ def collate_fn_pad(batch):
 
     return res_dict
 class HDF5Dataset(Dataset):
-    def __init__(self, directory, eval = False):
+    def __init__(self, directory, eval = False, leaderboard_version=1):
         '''
         directory: the directory of the dataset
         eval: if True, use the eval index
@@ -56,10 +56,14 @@ def __init__(self, directory, eval = False):
 
         self.eval_index = False
         if eval:
-            if not os.path.exists(os.path.join(self.directory, 'index_eval.pkl')):
+            index_file_name = 'index_eval.pkl'
+            if leaderboard_version == 2:
+                print("Using index to leaderboard version 2!!")
+                index_file_name = 'index_eval_v2.pkl'
+            if not os.path.exists(os.path.join(self.directory, index_file_name)):
                 raise Exception(f"No eval index file found! Please check {self.directory}")
             self.eval_index = eval
-            with open(os.path.join(self.directory, 'index_eval.pkl'), 'rb') as f:
+            with open(os.path.join(self.directory, index_file_name), 'rb') as f:
                 self.eval_data_index = pickle.load(f)
 
         self.scene_id_bounds = {}  # 存储每个scene_id的最大最小timestamp和位置
@@ -140,9 +144,9 @@ def __getitem__(self, index_):
                 res_dict['ego_motion'] = ego_motion
 
             if self.eval_index:
-                eval_mask = torch.tensor(f[key]['eval_mask'][:])
+                # looks like v2 not follow the same rule as v1 with eval_mask provided
+                eval_mask = torch.tensor(f[key]['eval_mask'][:]) if 'eval_mask' in f[key] else torch.ones_like(pc0[:, 0], dtype=torch.bool)
                 res_dict['eval_mask'] = eval_mask
-
         return res_dict
 
 if __name__ == "__main__":

scripts/network/dataloader.py

@@ -56,26 +56,29 @@ def __init__(self, cfg, eval=False):
 
         self.metrics = OfficialMetrics()
 
-        if 'checkpoint' in cfg:
-            self.load_checkpoint_path = cfg.checkpoint
+        self.load_checkpoint_path = cfg.checkpoint if 'checkpoint' in cfg else None
 
+
+        self.leaderboard_version = cfg.leaderboard_version if 'leaderboard_version' in cfg else 1
+        # NOTE(Qingwen): since we have seflow version which is unsupervised, we need to set the flag to false.
+        self.supervised_flag = cfg.supervised_flag if 'supervised_flag' in cfg else True
+        self.save_res = False
         if 'av2_mode' in cfg:
             self.av2_mode = cfg.av2_mode
-            self.save_res = cfg.save_res
-            if self.save_res:
+            self.save_res = cfg.save_res if 'save_res' in cfg else False
+            
+            if self.save_res or self.av2_mode == 'test':
                 self.save_res_path = Path(cfg.dataset_path).parent / "results" / cfg.output
                 os.makedirs(self.save_res_path, exist_ok=True)
-                print(f"We are in {cfg.av2_mode}, results will be saved in: {self.save_res_path}")
+                print(f"We are in {cfg.av2_mode}, results will be saved in: {self.save_res_path} with version: {self.leaderboard_version} format for online leaderboard.")
         else:
             self.av2_mode = None
             if 'pretrained_weights' in cfg:
                 if cfg.pretrained_weights is not None:
                     self.model.load_from_checkpoint(cfg.pretrained_weights)
 
-        if 'dataset_path' in cfg:
-            self.dataset_path = cfg.dataset_path
-        if 'res_name' in cfg:
-            self.vis_name = cfg.res_name
+        self.dataset_path = cfg.dataset_path if 'dataset_path' in cfg else None
+        self.vis_name = cfg.res_name if 'res_name' in cfg else 'default'
         self.save_hyperparameters()
 
     def training_step(self, batch, batch_idx):
@@ -130,14 +133,24 @@ def train_validation_step_(self, batch, res_dict):
                 self.metrics.step(v1_dict, v2_dict)
         else:
             pass
-        
+
+    def configure_optimizers(self):
+        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
+        return optimizer
+
+    def on_train_epoch_start(self):
+        self.time_start_train_epoch = time.time()
+
+    def on_train_epoch_end(self):
+        self.log("pre_epoch_cost (mins)", (time.time()-self.time_start_train_epoch)/60.0, on_step=False, on_epoch=True, sync_dist=True)
+    
     def on_validation_epoch_end(self):
         self.model.timer.print(random_colors=False, bold=False)
 
         if self.av2_mode == 'test':
             print(f"\nModel: {self.model.__class__.__name__}, Checkpoint from: {self.load_checkpoint_path}")
-            print(f"Test results saved in: {self.save_res_path}, Please run submit to zip the results and upload to online leaderboard.")
-            output_file = zip_res(self.save_res_path)
+            print(f"Test results saved in: {self.save_res_path}, Please run submit to zip the results and upload to online leaderboard. You processed to {self.leaderboard_version} version.")
+            output_file = zip_res(self.save_res_path, leaderboard_version=self.leaderboard_version)
             # wandb.log_artifact(output_file)
             return
 
@@ -157,31 +170,29 @@ def on_validation_epoch_end(self):
         self.metrics.print()
 
         self.metrics = OfficialMetrics()
+
+        if self.save_res:
+            print(f"We already write the flow_est into the dataset, please run following commend to visualize the flow. Copy and paste it to your terminal:")
+            print(f"python tests/scene_flow.py --flow_mode '{self.vis_name}' --data_dir {self.dataset_path}")
+            print(f"Enjoy! ^v^ ------ \n")
 
     def eval_only_step_(self, batch, res_dict):
-        batch = {key: batch[key][0] for key in batch if len(batch[key])>0}
-        res_dict = {key: res_dict[key][0] for key in res_dict if len(res_dict[key])>0}
-
         eval_mask = batch['eval_mask'].squeeze()
         pc0 = batch['origin_pc0']
         pose_0to1 = cal_pose0to1(batch["pose0"], batch["pose1"])
         transform_pc0 = pc0 @ pose_0to1[:3, :3].T + pose_0to1[:3, 3]
         pose_flow = transform_pc0 - pc0
 
+        final_flow = pose_flow.clone()
         if 'pc0_valid_point_idxes' in res_dict:
             valid_from_pc2res = res_dict['pc0_valid_point_idxes']
 
             # flow in the original pc0 coordinate
             pred_flow = pose_flow[~batch['gm0']].clone()
-            pred_flow[valid_from_pc2res] = pose_flow[~batch['gm0']][valid_from_pc2res] + res_dict['flow']
-
-            final_flow = pose_flow.clone()
+            pred_flow[valid_from_pc2res] = res_dict['flow'] + pose_flow[~batch['gm0']][valid_from_pc2res]
             final_flow[~batch['gm0']] = pred_flow
-        # else:
-        #     # pose_flow = pose_flows
-        #     # pred_flow_ = res_dict['flow'].cpu().detach()
-        #     # TODO: for other methods.... 
-        #     pred_flow = pose_flows.clone()
+        else:
+            final_flow[~batch['gm0']] = res_dict['flow'] + pose_flow[~batch['gm0']]
 
         if self.av2_mode == 'val': # since only val we have ground truth flow to eval
             gt_flow = batch["flow"]
@@ -199,59 +210,52 @@ def eval_only_step_(self, batch, res_dict):
             rigid_flow = pose_flow[eval_mask, :3].cpu().detach().numpy()
             is_dynamic = np.linalg.norm(save_pred_flow - rigid_flow, axis=1, ord=2) >= 0.05
             sweep_uuid = (batch['scene_id'], batch['timestamp'])
-            write_output_file(save_pred_flow, is_dynamic, sweep_uuid, self.save_res_path)
+            if self.leaderboard_version == 2:
+                save_pred_flow = (final_flow - pose_flow).cpu().detach().numpy() # all points here... since 2rd version we need to save the relative flow.
+            write_output_file(save_pred_flow, is_dynamic, sweep_uuid, self.save_res_path, leaderboard_version=self.leaderboard_version)
+
+    def run_model_wo_ground_data(self, batch):
+        # NOTE (Qingwen): only needed when val or test mode, since train we will go through collate_fn to remove.
+        batch['origin_pc0'] = batch['pc0'].clone()
+        batch['pc0'] = batch['pc0'][~batch['gm0']].unsqueeze(0)
+        batch['pc1'] = batch['pc1'][~batch['gm1']].unsqueeze(0)
+        if 'pcb0' in batch:
+            batch['pcb0'] = batch['pcb0'][~batch['gmb0']].unsqueeze(0)
+        self.model.timer[12].start("One Scan")
+        res_dict = self.model(batch)
+        self.model.timer[12].stop()
 
+        # NOTE (Qingwen): Since val and test, we will force set batch_size = 1 
+        batch = {key: batch[key][0] for key in batch if len(batch[key])>0}
+        res_dict = {key: res_dict[key][0] for key in res_dict if len(res_dict[key])>0}
+        return batch, res_dict
+    
     def validation_step(self, batch, batch_idx):
         if self.av2_mode == 'val' or self.av2_mode == 'test':
-            batch['origin_pc0'] = batch['pc0'].clone()
-            batch['pc0'] = batch['pc0'][~batch['gm0']].unsqueeze(0)
-            batch['pc1'] = batch['pc1'][~batch['gm1']].unsqueeze(0)
-            self.model.timer[12].start("One Scan")
-            res_dict = self.model(batch)
-            self.model.timer[12].stop()
+            batch, res_dict = self.run_model_wo_ground_data(batch)
             self.eval_only_step_(batch, res_dict)
         else:
             res_dict = self.model(batch)
             self.train_validation_step_(batch, res_dict)
 
-    def configure_optimizers(self):
-        optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
-        return optimizer
-
-    def on_train_epoch_start(self):
-        self.time_start_train_epoch = time.time()
-
-    def on_train_epoch_end(self):
-        self.log("pre_epoch_cost (mins)", (time.time()-self.time_start_train_epoch)/60.0, on_step=False, on_epoch=True, sync_dist=True)
-    
     def test_step(self, batch, batch_idx):
-        # NOTE (Qingwen): again, val and test we only allow batch_size = 1
-        batch['origin_pc0'] = batch['pc0'].clone()
-        batch['pc0'] = batch['pc0'][~batch['gm0']].unsqueeze(0)
-        batch['pc1'] = batch['pc1'][~batch['gm1']].unsqueeze(0)
-        res_dict = self.model(batch)
-        batch = {key: batch[key][0] for key in batch if len(batch[key])>0}
-        res_dict = {key: res_dict[key][0] for key in res_dict if len(res_dict[key])>0}
-
+        batch, res_dict = self.run_model_wo_ground_data(batch)
         pc0 = batch['origin_pc0']
         pose_0to1 = cal_pose0to1(batch["pose0"], batch["pose1"])
         transform_pc0 = pc0 @ pose_0to1[:3, :3].T + pose_0to1[:3, 3]
         pose_flow = transform_pc0 - pc0
 
+        final_flow = pose_flow.clone()
         if 'pc0_valid_point_idxes' in res_dict:
             valid_from_pc2res = res_dict['pc0_valid_point_idxes']
 
             # flow in the original pc0 coordinate
             pred_flow = pose_flow[~batch['gm0']].clone()
             pred_flow[valid_from_pc2res] = pose_flow[~batch['gm0']][valid_from_pc2res] + res_dict['flow']
 
-            final_flow = pose_flow.clone()
             final_flow[~batch['gm0']] = pred_flow
-        # else:
-        #     # pose_flow = pose_flows
-        #     # pred_flow_ = res_dict['flow'].cpu().detach()
-        #     # TODO: for other methods.... 
-        #     pred_flow = pose_flows.clone()
+        else:
+            final_flow[~batch['gm0']] = res_dict['flow'] + pose_flow[~batch['gm0']]
 
         # write final_flow into the dataset.
         key = str(batch['timestamp'])
@@ -262,7 +266,8 @@ def test_step(self, batch, batch_idx):
             f[key].create_dataset(self.vis_name, data=final_flow.cpu().detach().numpy().astype(np.float32))
 
     def on_test_epoch_end(self):
+        self.model.timer.print(random_colors=False, bold=False)
         print(f"\n\nModel: {self.model.__class__.__name__}, Checkpoint from: {self.load_checkpoint_path}")
-        print(f"We already write the estimate flow: {self.vis_name} into the dataset, please run following commend to visualize the flow. Copy and paste it to your terminal:")
+        print(f"We already write the flow_est into the dataset, please run following commend to visualize the flow. Copy and paste it to your terminal:")
         print(f"python tests/scene_flow.py --flow_mode '{self.vis_name}' --data_dir {self.dataset_path}")
         print(f"Enjoy! ^v^ ------ \n")