accept dense sampled maps if available

.dockerignore

@@ -1,6 +1,7 @@
-*~
-build
-stash
-old
-templates
-Dockerfile
+*
+!src/*.cpp
+!src/*.h
+!src/*.c
+!src/*.proto
+!src/CMakeLists.txt
+!CMakeLists.txt

CMakeLists.txt

@@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 2.6)
 
 PROJECT(makesweet)
 
+set (CMAKE_CXX_STANDARD 11)
+
 SET(BUILD_SHARED_LIBS ON)
 
 # Find YARP.  Point the YARP_DIR environment variable at your build.

Dockerfile

@@ -1,5 +1,7 @@
 FROM ubuntu:18.04
 
+# this is a very chubby docker image, it could be stripped down a lot.
+
 RUN \
   apt-get update; \
   apt-get install -y build-essential
@@ -19,14 +21,22 @@ RUN \
   cmake -DSKIP_ACE=TRUE ../yarp-*; \
   make 
 
+RUN \
+  apt-get update; \
+  apt-get install -y protobuf-compiler libprotobuf-dev
+
+RUN \
+  apt-get update; \
+  apt-get install -y libopencv-videoio-dev
+
 COPY . /makesweet/
 
 RUN \
   cd /makesweet; \
   mkdir build; \
   cd build; \
-  cmake -DYARP_DIR=/tmp/yarp ..; \
-  make
+  cmake -DUSE_OPENCV=ON -DUSE_DETAIL=ON -DYARP_DIR=/tmp/yarp ..; \
+  make VERBOSE=1
 
 RUN \
   echo "#!/bin/bash" > /reanimator; \

src/CMakeLists.txt

@@ -4,20 +4,41 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
 option(USE_OPENCV "Implement options that rely on OpenCV" FALSE)
 
+OPTION(USE_DETAIL "Use detailed sample maps rather than single-pixel summaries" OFF)
+
+IF(USE_DETAIL)
+  # Protobuf!
+  find_package(Protobuf REQUIRED)
+  include_directories(${PROTOBUF_INCLUDE_DIRS})
+  include_directories(${CMAKE_BINARY_DIR})
+  PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS frame.proto)
+  ADD_DEFINITIONS(-DUSE_DETAIL=1)
+ELSE()
+  SET(PROTO_SRCS)
+  SET(PROTO_HDRS)
+ENDIF()
+
 ADD_LIBRARY(msmap msmap.h msmap.c)
-ADD_LIBRARY(filer Filer.cpp Filer.h)
-TARGET_LINK_LIBRARIES(filer gd zzip)
 
 ADD_EXECUTABLE(reanimator reanimator.cpp Prop.h Prop.cpp Pixer.h
-  Mapping.h Repository.h Repository.cpp Render.h Render.cpp
+  Mapping.h Mapping.cpp Repository.h Repository.cpp Render.h Render.cpp
   Input.h Input.cpp Inputs.h Renders.h Renders.cpp
   GifAnim.h GifAnim.cpp
   VidAnim.h VidAnim.cpp
-  gd_topal.c)
-TARGET_LINK_LIBRARIES(reanimator filer)
+  gd_topal.c ${PROTO_SRCS} ${PROTO_HDRS}
+  Filer.cpp Filer.h)
+# TARGET_LINK_LIBRARIES(reanimator filer)
+TARGET_LINK_LIBRARIES(reanimator gd zzip)
 
 if (USE_OPENCV)
   add_definitions("-DMAKESWEET_USE_OPENCV")
+  option(NEED_VIDEOIO "Need to link videoio in this version of opencv" ON)
   TARGET_LINK_LIBRARIES(reanimator opencv_core opencv_highgui opencv_imgproc)
+  if (NEED_VIDEOIO)
+    TARGET_LINK_LIBRARIES(reanimator opencv_videoio)
+  endif()
 endif()
 
+if (USE_DETAIL)
+  TARGET_LINK_LIBRARIES(reanimator ${PROTOBUF_LIBRARIES})
+endif()

src/Filer.h

@@ -17,6 +17,8 @@ public:
      if (src_zip) unload_zip();
    }
 
+   void *getZip() { return src_zip; }
+
    bool load_zip(const char *fname);
    bool unload_zip();
 

src/Mapping.cpp

@@ -0,0 +1,83 @@
+#include "Mapping.h"
+
+#include <fstream>
+#include <iostream>
+
+#ifdef USE_DETAIL
+#include <zzip/zzip.h>
+#include "src/frame.pb.h"
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#endif
+
+void Mapping::load_samples(const char *name, Filer& filer) {
+  if (name == NULL) { return; }
+#ifdef USE_DETAIL
+  have_sum = true;
+
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
+  makesweet::Frame frame;
+
+
+  ZZIP_DIR *zip = (ZZIP_DIR *)filer.getZip();
+  if (!zip) {
+    fprintf(stderr, "no zip file\n");
+    exit(1);
+  }
+  ZZIP_FILE *fp = NULL;
+  fp = zzip_file_open(zip, name, ZZIP_CASELESS);
+  if (!fp) {
+    fprintf(stderr,"Cannot open %s in zip\n", name);
+    exit(1);
+  }
+  char buf[10000];
+  std::string data;
+  while (true) {
+    zzip_ssize_t len = zzip_file_read(fp, buf, 10000);
+    if (!len) {
+      break;
+    }
+    data += std::string(buf, len);
+  }
+  zzip_file_close(fp);
+
+  printf("Loading samples %s starting\n", name);
+  google::protobuf::io::ArrayInputStream input_stream(data.data(), (int)data.size());
+  google::protobuf::io::CodedInputStream coded_stream(&input_stream);
+  coded_stream.SetTotalBytesLimit(2147483647, -1);
+  // if (!frame.ParseFromString(data)) {
+  if (!frame.ParseFromCodedStream(&coded_stream)) {
+    fprintf(stderr, "cannot load samples %s\n", name);
+    exit(1);
+  }
+  // std::cout << frame.DebugString() << std::endl;
+  float min_fx = frame.min_fx();
+  float max_fx = frame.max_fx();
+  float min_fy = frame.min_fy();
+  float max_fy = frame.max_fy();
+  float fscale = frame.fscale();
+  printf("Loading samples %s summarizing\n", name);
+  for (auto &summary: frame.summaries()) {
+    int x = summary.x();
+    int y = summary.y();
+    Summary& lsummary = sum.summaries[sum.offset(x, y)];
+    /*
+    printf("samples %d\n", (int)summary.samples().size());
+    if (summary.samples().size() == 80) {
+      for (auto &sample: summary.samples()) {
+        printf("%.6f %.6f %.2f   ", sample.fx(), sample.fy(), sample.factor());
+      }
+      printf("\n");
+    }
+    */
+    for (auto &sample: summary.samples()) {
+      float fx = ((sample.fx() / fscale) * (max_fx - min_fx) + min_fx);
+      float fy = ((sample.fy() / fscale) * (max_fy - min_fy) + min_fy);
+      float factor = (float)sample.factor();
+      // printf(" %g %g %g\n", (double) fx, (double) fy, (double) factor);
+      lsummary.samples.push_back(Sample({fx, fy, factor}));
+    }
+  }
+  printf("Loading samples %s done\n", name);
+#endif
+}

src/Mapping.h

@@ -3,18 +3,46 @@
 
 #include <yarp/sig/Image.h>
 #include <string>
+#include <map>
+#include <vector>
+#include "Filer.h"
+
+struct Sample {
+  float fx;
+  float fy;
+  float factor;
+};
+
+class Summary {
+public:
+  std::vector<Sample> samples;
+};
+
+class Summaries {
+public:
+  int offset(int x, int y) const {
+    return 10000 * y + x;
+  }
+
+  std::map<int, Summary> summaries;
+};
 
 class Mapping {
 public:
   yarp::sig::ImageOf<yarp::sig::PixelBgra> map1, map2, light, dark, neutral;
   std::string map1_name, map2_name, light_name, dark_name, neutral_name;
+  Summaries sum;
+  bool have_sum;
 
   // scale factor for interpreting map
   int scale;
 
   Mapping() {
     scale = 1;
+    have_sum = false;
   }
+
+  void load_samples(const char *name, Filer& filer);
 };
 
 #endif

src/Render.cpp

@@ -33,6 +33,105 @@ void Render::check() {
   }
 }
 
+void Render::add_sampled(const Input& in) {
+  check();
+  double active_scale = in.in_scale;
+  active_scale /= 2; //mapping->scale;
+  int off = (mapping->map2.height()-mapping->light.height())/2;
+
+  if (mapping->map2.width()<mapping->light.width()) {
+    return;
+  }
+
+  Summary empty;
+  IMGFOR(out,x,y) {
+    PixelBgra& pix = out(x,y);
+    const PixelBgra& lightPixel = mapping->light(x,y);
+    const PixelBgra& darkPixel = mapping->dark(x,y);
+    const PixelBgra& selPixel = mapping->map2(x,y+off);
+    auto it = mapping->sum.summaries.find(mapping->sum.offset(x, mapping->light.height() - 1 - y));
+    const Summary& summary = (it != mapping->sum.summaries.end()) ? it->second : empty;
+
+    PixelBgra result, m;
+    bool d = (selPixel.r==in.layer);
+    if (d) {
+      m.a = 255;
+      m.r = 255;
+      m.g = 0;
+      m.b = 0;
+      float ct = 0;
+      double aa = 0, rr = 0, gg = 0, bb = 0;
+      for (auto &sample: summary.samples) {
+        // printf(">>> %g %g\n", sample.fx, sample.fy);
+        if (sample.fx < 0 || sample.fx > 1 || sample.fy < 0 || sample.fy > 1) {
+          continue;
+        }
+
+        int x = (int)(sample.fx*4096+4096+0.5);
+        int y = (int)(sample.fy*4096+4096+0.5);
+        if (x>4095) x -= 4096;
+        if (y>4095) y -= 4096;
+        if (x>4095) x = 4095;
+        if (y>4095) y = 4095;
+        if (x<0) x = 0;
+        if (y<0) y = 0;
+        y = 4095-y;
+
+        double x1 = x - RR;
+        double y1 = y - RR;
+        x1 *= in.xs;
+        y1 *= in.ys;
+        double xx =  in.xa*x1 + in.ya*y1;
+        double yy = -in.ya*x1 + in.xa*y1;
+        xx += RR + in.xo;
+        yy += RR + in.yo;
+        xx = in.in_x0 + active_scale*xx/RR;
+        yy = in.in_y0 + active_scale*yy/RR;
+
+        const PixelBgra& m2 = in.get().safePixel((int)xx,(int)yy); 
+        m.r = m2.r;
+        m.g = m2.g;
+        m.b = m2.b;
+
+        float f = sample.factor;
+        aa += m2.a * f;
+        rr += m2.a * m2.r * f;
+        gg += m2.a * m2.g * f;
+        bb += m2.a * m2.b * f;
+        ct += f;
+      }
+      if (aa < 0.001) aa = 0.001;
+      if (ct < 0.001) ct = 0.001;
+      rr /= aa;
+      gg /= aa;
+      bb /= aa;
+      aa /= ct;
+      m.r = int(rr);
+      m.g = int(gg);
+      m.b = int(bb);
+      m.a = int(aa);
+
+      result.r = darkPixel.r + ((lightPixel.r-darkPixel.r)*m.r)/255;
+      result.g = darkPixel.g + ((lightPixel.g-darkPixel.g)*m.g)/255;
+      result.b = darkPixel.b + ((lightPixel.b-darkPixel.b)*m.b)/255;
+      result.a = m.a;
+      if (darkPixel.a<result.a) {
+	result.a = darkPixel.a;
+      }
+      if (result.a>0) {
+	PixelBgra& outPixel = out.pixel(x,y);
+	if (result.a>250) {
+	  outPixel = result;
+	} else {
+	  outPixel.r += ((result.r-outPixel.r)*result.a)/255;
+	  outPixel.g += ((result.g-outPixel.g)*result.a)/255;
+	  outPixel.b += ((result.b-outPixel.b)*result.a)/255;
+	}
+      }
+    }
+  }
+}
+
 void Render::add(const Input& in) {
   check();
   double active_scale = in.in_scale;
@@ -419,7 +518,11 @@ void Render::apply_scaled(const Inputs& ins, int w, int h) {
   const std::vector<Input>& data = ins.get();
   for (std::vector<Input>::const_iterator it = data.begin();
        it != data.end(); it++) {
-    add(*it);
+    if (mapping->have_sum) {
+      add_sampled(*it);
+    } else {
+      add(*it);
+    }
   }
   post();
   if (w>0&&h>0 && (w!=out.width()||h!=out.height())) {

src/Render.h

@@ -25,12 +25,14 @@ private:
   const Mapping *mapping;
   yarp::sig::ImageOf<yarp::sig::PixelBgra> out;
   yarp::sig::ImageOf<yarp::sig::PixelBgra> out_scaled;
+  std::string quality;
 
   // active variables
 
 public:
-  Render() {
+  Render(const std::string& nquality = "none") {
     mapping = 0 /*NULL*/;
+    quality = nquality;
   }
 
   void check();
@@ -50,6 +52,8 @@ public:
 
   void add(const Input& in);
 
+  void add_sampled(const Input& in);
+
   void apply(const Inputs& ins) {
     apply_scaled(ins,-1,-1);
   }

src/Renders.cpp

@@ -25,7 +25,7 @@ Render *Renders::get_render(int index) {
   check();
   map<int,Render>::iterator it = renders.find(index);
   if (it == renders.end()) {
-    Render& render = renders[index] = Render();
+    Render& render = renders[index] = Render(quality);
     render.attach_mapping(repo->get_mapping(index));
     render.apply_scaled(*inputs,w,h);
     int rct = (int)renders.size();
@@ -41,6 +41,7 @@ void Renders::remove_render(int index) {
   if (it!=renders.end()) {
     renders.erase(it);
   }
+  repo->remove_mapping(index);
 }
 
 class Obs {
@@ -70,7 +71,7 @@ bool is_active(const yarp::sig::PixelBgra& pix) {
 Obs tweakOne(const yarp::sig::ImageOf<yarp::sig::PixelBgra>& img,
              float x0, float y0, float x1, float y1, float dx, float dy,
              float xt, float yt, float ix, float iy,
-             cv::Subdiv2D subdiv, float scale, float sx, float sy) {
+             cv::Subdiv2D subdiv, float scale) {
   int w = img.width();
   int h = img.height();
   Obs obs;
@@ -230,7 +231,7 @@ void Renders::scan(int frame) {
   int ins = inputs->get().size();
   for (int k=0; k<ins; k++) {
     printf("Scan input %d frame %d\n", k, frame);
-    Render render;
+    Render render(quality);
     render.attach_mapping(repo->get_mapping(frame));
     std::vector<CloudPoint> pts;
     Input& in = inputs->get_mod()[k];
@@ -303,7 +304,7 @@ void Renders::scan(int frame) {
 
 bool Renders::auto_zoom() {
   check();
-  Render render;
+  Render render(quality);
   render.attach_mapping(repo->get_mapping(0));
   return render.auto_zoom(*inputs);
 }

src/Renders.h

@@ -12,13 +12,15 @@ private:
   std::map<int,Render> renders;
   int w, h;
   int peak_cache_count;
+  std::string quality;
 
 public:
-  Renders() {
+  Renders(const std::string& nquality) {
     inputs = 0 /*NULL*/;
     repo = 0 /*NULL*/;
     w = h = -1;
     peak_cache_count = 0;
+    quality = nquality;
   }
 
   void check();

src/Repository.cpp

@@ -16,12 +16,14 @@ void Repository::load_part(const char *postfix, int idx) {
   string neutral = base + "TransparentData";
   string map1 = base + "MapData";
   string map2 = base + "SelData";
+  string samples = base + "Samples";
   if (postfix[0]!='\0') {
     dark += postfix;
     light += postfix;
     neutral += postfix;
     map1 += postfix;
     map2 += postfix;
+    samples += postfix;
   }
 
   if (mappings.find(idx)==mappings.end()) {
@@ -40,12 +42,14 @@ void Repository::load_part(const char *postfix, int idx) {
   }
   
   bool have_neutral = inventory.check(neutral.c_str());
+  bool have_samples = inventory.check(samples.c_str());
 
   ConstString light_name = inventory.find(light.c_str()).asString();
   ConstString dark_name = inventory.find(dark.c_str()).asString();
   ConstString neutral_name = inventory.find(neutral.c_str()).asString();
   ConstString map1_name = inventory.find(map1.c_str()).asString();
   ConstString map2_name = inventory.find(map2.c_str()).asString();
+  ConstString samples_name = inventory.find(samples.c_str()).asString();
 
   dbg_printf("Loading light %s\n", light_name.c_str());
   if (!filer.load(light_name.c_str(),s.light)) exit(1);
@@ -53,9 +57,11 @@ void Repository::load_part(const char *postfix, int idx) {
   dbg_printf("Loading dark\n");
   if (!filer.load(dark_name.c_str(),s.dark)) exit(1);
   s.dark_name = dark_name;
-  dbg_printf("Loading map1\n");
-  if (!filer.load(map1_name.c_str(),s.map1)) exit(1);
-  s.map1_name = map1_name;
+  if (!have_samples) {
+    dbg_printf("Loading map1\n");
+    if (!filer.load(map1_name.c_str(),s.map1)) exit(1);
+    s.map1_name = map1_name;
+  }
   dbg_printf("Loading map2\n");
   if (!filer.load(map2_name.c_str(),s.map2)) exit(1);
   s.map2_name = map2_name;
@@ -67,6 +73,12 @@ void Repository::load_part(const char *postfix, int idx) {
     s.neutral = s.light;
     s.neutral_name = light_name;
   }
+  if (have_samples) {
+    dbg_printf("Loading samples\n");
+    s.load_samples(samples_name.c_str(), filer);
+  } else {
+    s.load_samples(NULL, filer);
+  }
 
   s.scale = 1;
 }

src/VidAnim.cpp

@@ -60,6 +60,7 @@ void VidAnim::apply(const char *fname) {
       target = curr + step;
       Render *r = renders->get_render(i);
       v1.copy(r->get());
+      renders->remove_render(i);
       m = cvarrToMat(static_cast<const IplImage*>(v1.getIplImage()));
       base++;
     }

src/frame.proto

@@ -0,0 +1,24 @@
+syntax = "proto2";
+
+package makesweet;
+
+message Sample {
+  required int32 fx = 1;
+  required int32 fy = 2;
+  required int32 factor = 3;
+}
+
+message Summary {
+  required int32 x = 1;
+  required int32 y = 2;
+  repeated Sample samples = 3;
+}
+
+message Frame {
+  repeated Summary summaries = 1;
+  required float min_fx = 2;
+  required float max_fx = 3;
+  required float min_fy = 4;
+  required float max_fy = 5;
+  required int32 fscale = 6;
+}

src/msmap.c

@@ -1,3 +1,4 @@
+// MY GOODNESS this is crufty how did this get in here.
 
 #include "msmap.h"
 

src/reanimator.cpp

@@ -68,7 +68,7 @@ int main(int argc, char *argv[]) {
     layer++;
   }
 
-  Renders renders;
+  Renders renders("sampled");
   renders.attach_repository(&repo);
   renders.attach_inputs(&ins);
   if (options.check("w")&&options.check("h")) {