diff --git a/pdal/io/NumpyReader.cpp b/pdal/io/NumpyReader.cpp index 47992d8f..8983340e 100644 --- a/pdal/io/NumpyReader.cpp +++ b/pdal/io/NumpyReader.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include "../plang/Environment.hpp" @@ -407,10 +408,13 @@ void NumpyReader::createFields(PointLayoutPtr layout) offset = PyLong_AsLong(offset_o); // Get type. - type = getType((PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0), - name); + PyArray_Descr* dt = (PyArray_Descr *)PySequence_Fast_GET_ITEM(tup, 0); + type = getType(dt, name); + + char byteorder = dt->byteorder; + int elsize = dt->elsize; id = registerDim(layout, name, type); - m_fields.push_back({id, type, offset}); + m_fields.push_back({id, type, offset, byteorder, elsize}); } } } @@ -510,6 +514,12 @@ void NumpyReader::ready(PointTableRef table) for (npy_intp i = 0; i < m_ndims; ++i) log()->get(LogLevel::Debug) << "numpy shape dimension number '" << i << "' is '" << m_shape[i] <<"'" << std::endl; + + PointLayoutPtr layout = table.layout(); + MetadataNode m = layout->toMetadata(); + + pdal::Utils::toJSON(m, log()->get(LogLevel::Debug3)); + } bool NumpyReader::nextPoint() @@ -530,12 +540,93 @@ bool NumpyReader::nextPoint() } + + bool NumpyReader::loadPoint(PointRef& point, point_count_t position) { using namespace Dimension; + pdal::SwitchableExtractor extractor(p_data, *m_strideptr); + + std::vector buf(*m_strideptr,0); + + float flt(0.0); + double dbl(0.0); + uint8_t u8(0); + uint16_t u16(0); + uint32_t u32(0); + uint64_t u64(0); + int8_t i8(0); + int16_t i16(0); + int32_t i32(0); + int64_t i64(0); + for (const Field& f : m_fields) - point.setField(f.m_id, f.m_type, (void*)(p_data + f.m_offset)); + { + if (f.m_byteorder == '>') + extractor.switchToBigEndian(); + else + extractor.switchToLittleEndian(); + + switch (f.m_type) + { + case Dimension::Type::Signed8: + extractor >> i8; + point.setField(f.m_id, i8); + break; + + case Dimension::Type::Signed16: + extractor >> i16; + point.setField(f.m_id, i16); + break; + + case Dimension::Type::Signed32: + extractor >> i32; + point.setField(f.m_id, i32); + break; + + case Dimension::Type::Signed64: + extractor >> i64; + point.setField(f.m_id, i64); + break; + + case Dimension::Type::Unsigned8: + extractor >> u8; + point.setField(f.m_id, u8); + break; + + case Dimension::Type::Unsigned16: + extractor >> u16; + point.setField(f.m_id, u16); + break; + + case Dimension::Type::Unsigned32: + extractor >> u32; + point.setField(f.m_id, u32); + break; + + case Dimension::Type::Unsigned64: + extractor >> u64; + point.setField(f.m_id, u64); + break; + + case Dimension::Type::Float: + extractor >> flt; + point.setField(f.m_id, flt); + break; + + case Dimension::Type::Double: + extractor >> dbl; + point.setField(f.m_id, dbl); + break; + + default: + // skip it + extractor.skip(f.m_elsize); + + } + + } if (m_storeXYZ) { diff --git a/pdal/io/NumpyReader.hpp b/pdal/io/NumpyReader.hpp index 1b8ae8dd..2c62a1d5 100644 --- a/pdal/io/NumpyReader.hpp +++ b/pdal/io/NumpyReader.hpp @@ -120,6 +120,8 @@ class PDAL_DLL NumpyReader : public Reader, public Streamable Dimension::Id m_id; Dimension::Type m_type; int m_offset; + char m_byteorder; + int m_elsize; }; std::vector m_fields; point_count_t m_index; diff --git a/pdal/test/NumpyReaderTest.cpp b/pdal/test/NumpyReaderTest.cpp index 048e600c..bd930093 100644 --- a/pdal/test/NumpyReaderTest.cpp +++ b/pdal/test/NumpyReaderTest.cpp @@ -236,3 +236,100 @@ TEST(NumpyReaderTest, threedim_col) } } + +TEST(NumpyReaderTest, threedim_row_script) +{ + StageFactory f; + + Options opts; + opts.add("filename", Support::datapath("threedim.py")); + opts.add("order", "row"); + opts.add("function", "load"); + opts.add("module", "threedim"); + opts.add("fargs", Support::datapath("threedim.npy")); + + NumpyReader reader; + reader.setOptions(opts); + + PointTable table; + + reader.prepare(table); + + PointViewSet viewSet = reader.execute(table); + PointViewPtr view = *viewSet.begin(); + PointLayoutPtr layout = view->layout(); + Dimension::Id health = layout->findDim("Health"); + Dimension::Id foo = layout->findDim("Foo"); + + EXPECT_EQ(view->size(), 24u); + int ex = 0; + int ey = 0; + int ez = 0; + for (PointId id = 0; id < view->size(); ++id) + { + int f = view->getFieldAs(foo, id); + int h = view->getFieldAs(health, id); + int i = view->getFieldAs(Dimension::Id::Intensity, id); + int x = view->getFieldAs(Dimension::Id::X, id); + int y = view->getFieldAs(Dimension::Id::Y, id); + int z = view->getFieldAs(Dimension::Id::Z, id); + EXPECT_EQ(f, ((int)id / 12) % 2); + EXPECT_EQ(h, ((int)id / 4) % 3); + EXPECT_EQ(i, (int)id % 4); + EXPECT_EQ(x, ex); + EXPECT_EQ(y, ey); + EXPECT_EQ(z, ez); + // Array is 2 x 3 x 4 + ez++; + if (ez == 4) + { + ez = 0; + ey++; + if (ey == 3) + { + ey = 0; + ex++; + } + } + } +} + +TEST(NumpyReaderTest, four_floats_script) +{ + StageFactory f; + + Options opts; + opts.add("filename", Support::datapath("four-floats.py")); + opts.add("order", "row"); + opts.add("function", "load"); + opts.add("module", "four_floats"); + opts.add("fargs", Support::datapath("four-floats-be.bin")); + + NumpyReader reader; + reader.setOptions(opts); + + PointTable table; + + reader.prepare(table); + + PointViewSet viewSet = reader.execute(table); + PointViewPtr view = *viewSet.begin(); + PointLayoutPtr layout = view->layout(); + + EXPECT_EQ(view->size(), 10u); + int ex = 0; + int ey = 0; + int ez = 0; + int factor = 10; + for (PointId id = 0; id < view->size(); ++id) + { + int t = view->getFieldAs(Dimension::Id::GpsTime, id); + int x = view->getFieldAs(Dimension::Id::X, id); + int y = view->getFieldAs(Dimension::Id::Y, id); + int z = view->getFieldAs(Dimension::Id::Z, id); + EXPECT_EQ(x, (int)id * 1 * factor); + EXPECT_EQ(y, (int)id * 2 * factor); + EXPECT_EQ(z, (int)id * 3 * factor); + EXPECT_EQ(t, (int)id * 4 * factor); + } +} diff --git a/pdal/test/data/four-floats-be.bin b/pdal/test/data/four-floats-be.bin new file mode 100644 index 00000000..032167df Binary files /dev/null and b/pdal/test/data/four-floats-be.bin differ diff --git a/pdal/test/data/four-floats.py b/pdal/test/data/four-floats.py new file mode 100644 index 00000000..38408a9c --- /dev/null +++ b/pdal/test/data/four-floats.py @@ -0,0 +1,10 @@ +import numpy as np + + +dtype = np.dtype([('X', '>f4'), ('Y', '>f4'), ('Z', 'f4')]) + +def load(filename): + data = open(filename, 'rb').read() + array = np.frombuffer(data, dtype=dtype) + + return array diff --git a/pdal/test/data/threedim.py b/pdal/test/data/threedim.py new file mode 100644 index 00000000..95f9a065 --- /dev/null +++ b/pdal/test/data/threedim.py @@ -0,0 +1,5 @@ +import numpy as np + +def load(filename): + array = np.load(filename) + return array