From 029d5128f3bf7a4fed96c4b587092707ee4c8908 Mon Sep 17 00:00:00 2001 From: Kin Date: Mon, 10 Mar 2025 08:55:38 +0100 Subject: [PATCH 1/5] docs(README): fix typo on readme and comments in code. --- README.md | 9 ++++++--- src/models/__init__.py | 2 ++ tools/visualization.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 782ed8a..d47bc95 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ International Conference on Robotics and Automation (**ICRA**) 2024 ๐Ÿ’ž If you find *OpenSceneFlow* useful to your research, please cite [our works ๐Ÿ“–](#cite-us) and give a star ๐ŸŒŸ as encouragement. (เฉญหŠ๊’ณโ€‹ห‹)เฉญโœง -๐ŸŽ One repository, All methods!. Additionally, *OpenSceneFlow* integrates the following excellent work: [ICLR'24 ZeroFlow](https://arxiv.org/abs/2305.10424), [ICCV'23 FastNSF](https://arxiv.org/abs/2304.09121), [RA-L'21 FastFlow](https://arxiv.org/abs/2103.01306), [NeurIPS'21 NSFP](https://arxiv.org/abs/2111.01253), +๐ŸŽ One repository, All methods! +Additionally, *OpenSceneFlow* integrates the following excellent work: [ICLR'24 ZeroFlow](https://arxiv.org/abs/2305.10424), [ICCV'23 FastNSF](https://arxiv.org/abs/2304.09121), [RA-L'21 FastFlow](https://arxiv.org/abs/2103.01306), [NeurIPS'21 NSFP](https://arxiv.org/abs/2111.01253). (More on the way)
Summary of them: @@ -42,7 +43,7 @@ International Conference on Robotics and Automation (**ICRA**) 2024
-๐Ÿ’ก: Want to learn how to add your own network in this structure? Check [Contribute section] and know more about the code. Fee free to pull request and your bibtex [here](#cite-us) by pull request. +๐Ÿ’ก: Want to learn how to add your own network in this structure? Check [Contribute section](assets/README.md#contribute) and know more about the code. Fee free to pull request and your bibtex [here](#cite-us) by pull request. --- @@ -90,7 +91,9 @@ cd /home/kin/workspace/OpenSceneFlow/assets/cuda/chamfer3D && /opt/conda/envs/op Refer to [dataprocess/README.md](dataprocess/README.md) for dataset download instructions. Currently, we support **Argoverse 2**, **Waymo**, and **custom datasets** (more datasets will be added in the future). -After downloading, convert the raw data to `.h5` format for easy training, evaluation, and visualization. Follow the steps in [dataprocess/README.md#process](dataprocess/README.md#process). For a quick start, use our **mini processed dataset**, which includes one scene in `train` and `val`. It is pre-converted to `.h5` format with label data ([Zenodo](https://zenodo.org/records/13744999/files/demo_data.zip)/[HuggingFace](https://huggingface.co/kin-zhang/OpenSceneFlow/blob/main/demo_data.zip)). +After downloading, convert the raw data to `.h5` format for easy training, evaluation, and visualization. Follow the steps in [dataprocess/README.md#process](dataprocess/README.md#process). + +For a quick start, use our **mini processed dataset**, which includes one scene in `train` and `val`. It is pre-converted to `.h5` format with label data ([Zenodo](https://zenodo.org/records/13744999/files/demo_data.zip)/[HuggingFace](https://huggingface.co/kin-zhang/OpenSceneFlow/blob/main/demo_data.zip)). ```bash diff --git a/src/models/__init__.py b/src/models/__init__.py index 347b5a6..edb65c0 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -7,6 +7,8 @@ # If you find this repo helpful, please cite the respective publication as # listed on the above website. """ +import warnings +warnings.simplefilter(action="ignore", category=FutureWarning) from .deflow import DeFlow from .fastflow3d import FastFlow3D diff --git a/tools/visualization.py b/tools/visualization.py index 31a157c..9cd6651 100644 --- a/tools/visualization.py +++ b/tools/visualization.py @@ -69,7 +69,7 @@ def check_flow( def vis( data_dir: str ="/home/kin/data/av2/preprocess/sensor/mini", - res_name: str = "flow", # "flow", "flow_est" + res_name: str = "flow", # any res_name we write before in HDF5Data start_id: int = 0, point_size: float = 2.0, mode: str = "vis", From 3ee0a559a0a53d3c2314d4be79df137a74c6d37b Mon Sep 17 00:00:00 2001 From: Kin Date: Mon, 10 Mar 2025 09:22:06 +0100 Subject: [PATCH 2/5] !fix(gt): expanding the bbx based on object speed for non-ego motion distortion in data. check more detail on pull request description. --- dataprocess/extract_av2.py | 39 ++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/dataprocess/extract_av2.py b/dataprocess/extract_av2.py index e397bab..188b03c 100644 --- a/dataprocess/extract_av2.py +++ b/dataprocess/extract_av2.py @@ -35,6 +35,7 @@ import pickle from zipfile import ZipFile import pandas as pd +from copy import deepcopy import os, sys BASE_DIR = os.path.abspath(os.path.join( os.path.dirname( __file__ ), '..' )) @@ -132,23 +133,49 @@ def compute_flow(sweeps, cuboids, poses): valid = np.ones(len(sweeps[0].xyz), dtype=np.bool_) # classes = -np.ones(len(sweeps[0].xyz), dtype=np.int8) classes = np.zeros(len(sweeps[0].xyz), dtype=np.uint8) + + # # old version + # for id in cuboids[0]: + # c0 = cuboids[0][id] + # c0.length_m += BOUNDING_BOX_EXPANSION # the bounding boxes are a little too tight and some points are missed + # c0.width_m += BOUNDING_BOX_EXPANSION + # obj_pts, obj_mask = c0.compute_interior_points(sweeps[0].xyz) + # classes[obj_mask] = CATEGORY_TO_INDEX[str(c0.category)] + # if id in cuboids[1]: + # c1 = cuboids[1][id] + # c1_SE3_c0 = c1.dst_SE3_object.compose(c0.dst_SE3_object.inverse()) + # obj_flow = c1_SE3_c0.transform_point_cloud(obj_pts) - obj_pts + # flow[obj_mask] = obj_flow.astype(np.float32) + # else: + # valid[obj_mask] = 0 + + # NOTE(HiMo): box expansion based on the object velocity + # check more detail: https://kin-zhang.github.io/HiMo for id in cuboids[0]: - c0 = cuboids[0][id] - c0.length_m += BOUNDING_BOX_EXPANSION # the bounding boxes are a little too tight and some points are missed - c0.width_m += BOUNDING_BOX_EXPANSION + c0 = deepcopy(cuboids[0][id]) obj_pts, obj_mask = c0.compute_interior_points(sweeps[0].xyz) - classes[obj_mask] = CATEGORY_TO_INDEX[str(c0.category)] - if id in cuboids[1]: c1 = cuboids[1][id] + c1_SE3_c0_ego_frame = ego1_SE3_ego0.inverse().compose(c1.dst_SE3_object.compose(c0.dst_SE3_object.inverse())) + rel_obj_flow = c1_SE3_c0_ego_frame.transform_point_cloud(obj_pts) - obj_pts + delta_move = abs(np.linalg.norm(rel_obj_flow, axis=0).mean()) + + if delta_move > 0.04: # only when it's moving + c0 = cuboids[0][id] + c0.length_m += (BOUNDING_BOX_EXPANSION + min(delta_move/2, 2)) # since 180/360 for two LiDARs orientation + c0.width_m += BOUNDING_BOX_EXPANSION + c0.height_m += BOUNDING_BOX_EXPANSION + obj_pts, obj_mask = c0.compute_interior_points(sweeps[0].xyz) + + # NOTE(Qingwen): after expansion, we need to recompute the flow c1_SE3_c0 = c1.dst_SE3_object.compose(c0.dst_SE3_object.inverse()) obj_flow = c1_SE3_c0.transform_point_cloud(obj_pts) - obj_pts + classes[obj_mask] = CATEGORY_TO_INDEX[str(c0.category)] flow[obj_mask] = obj_flow.astype(np.float32) else: valid[obj_mask] = 0 return flow, classes, valid, ego1_SE3_ego0 - sweeps = [Sweep.from_feather(data_dir / log_id / "sensors" / "lidar" / f"{ts}.feather") for ts in timestamps] # ================== Load annotations ================== From 700af39f0550d6e80131c2b3a9d753bc8675c3d3 Mon Sep 17 00:00:00 2001 From: Kin Date: Mon, 10 Mar 2025 10:32:03 +0100 Subject: [PATCH 3/5] docs(readme): fix typo on README. * tested successfully on docker things also. --- Dockerfile | 1 - README.md | 26 +++++++++++++++++++------- conf/save.yaml | 2 +- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a3cda8..aee21ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,4 +32,3 @@ RUN apt-get update && apt-get install libgl1 -y RUN cd /home/kin/workspace/OpenSceneFlow && /opt/conda/bin/mamba env create -f environment.yaml RUN cd /home/kin/workspace/OpenSceneFlow/assets/cuda/mmcv && /opt/conda/envs/opensf/bin/python ./setup.py install RUN cd /home/kin/workspace/OpenSceneFlow/assets/cuda/chamfer3D && /opt/conda/envs/opensf/bin/python ./setup.py install - diff --git a/README.md b/README.md index d47bc95..88afaad 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

OpenSceneFlow is a codebase for point cloud scene flow estimation. -It is also an official implementation of the following paper (sored by the time of publication): +It is also an official implementation of the following papers (sored by the time of publication): - **Flow4D: Leveraging 4D Voxel Network for LiDAR Scene Flow Estimation** *Jaeyeul Kim, Jungwan Woo, Ukcheol Shin, Jean Oh, Sunghoon Im* @@ -31,7 +31,7 @@ International Conference on Robotics and Automation (**ICRA**) 2024 ๐Ÿ’ž If you find *OpenSceneFlow* useful to your research, please cite [our works ๐Ÿ“–](#cite-us) and give a star ๐ŸŒŸ as encouragement. (เฉญหŠ๊’ณโ€‹ห‹)เฉญโœง ๐ŸŽ One repository, All methods! -Additionally, *OpenSceneFlow* integrates the following excellent work: [ICLR'24 ZeroFlow](https://arxiv.org/abs/2305.10424), [ICCV'23 FastNSF](https://arxiv.org/abs/2304.09121), [RA-L'21 FastFlow](https://arxiv.org/abs/2103.01306), [NeurIPS'21 NSFP](https://arxiv.org/abs/2111.01253). (More on the way) +Additionally, *OpenSceneFlow* integrates the following excellent work: [ICLR'24 ZeroFlow](https://arxiv.org/abs/2305.10424), [ICCV'23 FastNSF](https://arxiv.org/abs/2304.09121), [RA-L'21 FastFlow](https://arxiv.org/abs/2103.01306), [NeurIPS'21 NSFP](https://arxiv.org/abs/2111.01253). (More on the way...)
Summary of them: @@ -57,14 +57,16 @@ Additionally, *OpenSceneFlow* integrates the following excellent work: [ICLR'24 ## 0. Installation -**Environment**: Setup +There are two ways to install the codebase: directly on your [local machine](#environment-setup) or in a [Docker container](#docker-recommended-for-isolation). + +### Environment Setup ```bash git clone --recursive https://github.com/KTH-RPL/OpenSceneFlow.git cd OpenSceneFlow && mamba env create -f environment.yaml ``` -CUDA package (need install nvcc compiler), the compile time is around 1-5 minutes: +CUDA package (we already install nvcc compiler inside conda env), the compile time is around 1-5 minutes: ```bash mamba activate opensf # CUDA already install in python environment. I also tested others version like 11.3, 11.4, 11.7, 11.8 all works @@ -72,8 +74,9 @@ cd assets/cuda/mmcv && python ./setup.py install && cd ../../.. cd assets/cuda/chamfer3D && python ./setup.py install && cd ../../.. ``` -Or you always can choose [Docker](https://en.wikipedia.org/wiki/Docker_(software)) which isolated environment and free yourself from installation, you can pull it by. -If you have different arch, please build it by yourself `cd OpenSceneFlow && docker build -t zhangkin/opensf` by going through [build-docker-image](assets/README.md#build-docker-image) section. +### Docker (Recommended for Isolation) + +You always can choose [Docker](https://en.wikipedia.org/wiki/Docker_(software)) which isolated environment and free yourself from installation. Pull the pre-built Docker image or build manually. ```bash # option 1: pull from docker hub @@ -84,8 +87,11 @@ docker run -it --gpus all -v /dev/shm:/dev/shm -v /home/kin/data:/home/kin/data # and better to read your own gpu device info to compile the cuda extension again: cd /home/kin/workspace/OpenSceneFlow/assets/cuda/mmcv && /opt/conda/envs/opensf/bin/python ./setup.py install cd /home/kin/workspace/OpenSceneFlow/assets/cuda/chamfer3D && /opt/conda/envs/opensf/bin/python ./setup.py install + +mamba activate opensf ``` +If you prefer to build the Docker image by yourself, Check [build-docker-image](assets/README.md#build-docker-image) section for more details. ## 1. Data Preparation @@ -98,13 +104,19 @@ For a quick start, use our **mini processed dataset**, which includes one scene ```bash wget https://huggingface.co/kin-zhang/OpenSceneFlow/resolve/main/demo_data.zip -unzip demo_data.zip -p /home/kin/data/av2 +unzip demo_data.zip -d /home/kin/data/av2/h5py ``` Once extracted, you can directly use this dataset to run the [training script](#2-quick-start) without further processing. ## 2. Quick Start +Don't forget to active Python environment before running the code. + +```bash +mamba activate opensf +``` + ### Flow4D Train Flow4D with the leaderboard submit config. [Runtime: Around 18 hours in 4x RTX 3090 GPUs.] diff --git a/conf/save.yaml b/conf/save.yaml index 253843e..de1f5f5 100644 --- a/conf/save.yaml +++ b/conf/save.yaml @@ -1,4 +1,4 @@ -dataset_path: /home/kin/data/av2/preprocess_v2/demo/sensor/val +dataset_path: /home/kin/data/av2/h5py/demo/sensor/val checkpoint: /home/kin/model_zoo/seflow_best.ckpt res_name: # if None will directly be the `model_name.ckpt` in checkpoint path From 54e30614f0dc1a5e9240ff25d964e54cc3c17659 Mon Sep 17 00:00:00 2001 From: Kin Date: Mon, 10 Mar 2025 10:52:34 +0100 Subject: [PATCH 4/5] fix(env): add c++ compiler into env and pathtools for potential err on run codes. --- README.md | 3 +++ environment.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 88afaad..cc7f3d4 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,9 @@ There are two ways to install the codebase: directly on your [local machine](#en ```bash git clone --recursive https://github.com/KTH-RPL/OpenSceneFlow.git cd OpenSceneFlow && mamba env create -f environment.yaml + +# You may need export your LD_LIBRARY_PATH with env lib +# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/kin/mambaforge/lib ``` CUDA package (we already install nvcc compiler inside conda env), the compile time is around 1-5 minutes: diff --git a/environment.yaml b/environment.yaml index 8d99068..21e7aa4 100644 --- a/environment.yaml +++ b/environment.yaml @@ -26,6 +26,8 @@ dependencies: - scikit-learn==1.3.2 - hdbscan - setuptools==69.5.1 + - gxx_linux-64==11.4.0 + - pathtools - pip: - open3d==0.18.0 - dztimer From 32f6918b070b0d489371ad637017c8030ce9c2e5 Mon Sep 17 00:00:00 2001 From: Kin Date: Mon, 10 Mar 2025 10:58:14 +0100 Subject: [PATCH 5/5] docs(bib): add himo into cite for reference on fixed flow gt. --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc7f3d4..2a17691 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ International Conference on Robotics and Automation (**ICRA**) 2024 [ Backbone ] [ Supervised ] - [ [arXiv](https://arxiv.org/abs/2401.16122) ] [ [Project](https://github.com/KTH-RPL/DeFlow) ] → [here](#deflow) -๐Ÿ’ž If you find *OpenSceneFlow* useful to your research, please cite [our works ๐Ÿ“–](#cite-us) and give a star ๐ŸŒŸ as encouragement. (เฉญหŠ๊’ณโ€‹ห‹)เฉญโœง +๐Ÿ’ž If you find *OpenSceneFlow* useful to your research, please cite [**our works** ๐Ÿ“–](#cite-us) and give a star ๐ŸŒŸ as encouragement. (เฉญหŠ๊’ณโ€‹ห‹)เฉญโœง ๐ŸŽ One repository, All methods! -Additionally, *OpenSceneFlow* integrates the following excellent work: [ICLR'24 ZeroFlow](https://arxiv.org/abs/2305.10424), [ICCV'23 FastNSF](https://arxiv.org/abs/2304.09121), [RA-L'21 FastFlow](https://arxiv.org/abs/2103.01306), [NeurIPS'21 NSFP](https://arxiv.org/abs/2111.01253). (More on the way...) +Additionally, *OpenSceneFlow* integrates following excellent works: [ICLR'24 ZeroFlow](https://arxiv.org/abs/2305.10424), [ICCV'23 FastNSF](https://arxiv.org/abs/2304.09121), [RA-L'21 FastFlow](https://arxiv.org/abs/2103.01306), [NeurIPS'21 NSFP](https://arxiv.org/abs/2111.01253). (More on the way...)
Summary of them: @@ -243,6 +243,12 @@ https://github.com/user-attachments/assets/07e8d430-a867-42b7-900a-11755949de21 pages={2105-2111}, doi={10.1109/ICRA57147.2024.10610278} } +@article{zhang2025himu, + title={HiMo: High-Speed Objects Motion Compensation in Point Cloud}, + author={Zhang, Qingwen and Khoche, Ajinkya and Yang, Yi and Ling, Li and Sina, Sharif Mansouri and Andersson, Olov and Jensfelt, Patric}, + year={2025}, + journal={arXiv preprint arXiv:2503.00803}, +} ``` And our excellent collaborators works as followings: