shampoo/gunkiss
shampoo
/
gunkiss
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use cv::Barcode for scanning instead of ZBar

This commit is contained in:
ohsqueezy 2022-09-26 16:13:42 -04:00
parent 862030b296
commit 59db4015b5
5 changed files with 125 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -94,7 +94,7 @@ linux : CFLAGS = -g -Wall -Wextra -O1 -c -I$(SB_LIB_DIR) -I$(SB_SRC_DIR) $(SDL_C
linux : CXXFLAGS = $(CFLAGS) --std=c++17
linux : LFLAGS = $(SDL_LFLAGS) -Wl,--enable-new-dtags -lpthread -lGL -lGLESv2 -lSDL2_image -lSDL2_ttf -lSDL2_mixer -lstdc++fs -lcurl \
-L$(HOME)/local/opencv/lib -Wl,-rpath,$(HOME)/local/opencv/lib -lopencv_videoio -lopencv_core -lopencv_highgui -lopencv_imgproc \
-L$(HOME)/local/zbar/lib -Wl,-rpath,$(HOME)/local/zbar/lib -lzbar
-l opencv_barcode -L$(HOME)/local/zbar/lib -Wl,-rpath,$(HOME)/local/zbar/lib -lzbar
linux : $(GLEW_DIR)glew.o $(addprefix $(SDLGFX2_DIR),SDL2_rotozoom.o SDL2_gfxPrimitives.o) \
$(SB_O_FILES) $(SRC_O_FILES)
$(CREATE_FONT_SYMLINK)
@ -107,11 +107,11 @@ linux : $(GLEW_DIR)glew.o $(addprefix $(SDLGFX2_DIR),SDL2_rotozoom.o SDL2_gfxPri
# Use Emscripten to output JavaScript and an HTML index page for running in the browser
EMSCRIPTENHOME = $(HOME)/ext/software/emsdk/upstream/emscripten
EMSCRIPTEN_CFLAGS = -O1 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" -s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 \
EMSCRIPTEN_CFLAGS = -O0 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" -s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 \
--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include -I $(HOME)/local/opencv/include/opencv4
EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main', '_malloc']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm/lib/,*.a)) \
$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/lib/,*.a)) \
$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a)) $(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \
--preload-file "src/shaders/"@/"src/shaders/"

12
config.json
View File

@ -59,9 +59,13 @@
"json-save-directory": "local/scans",
"barcode": "",
"capture-device": "/dev/video0",
"brightness-addition": 20,
"contrast-multiplication": 1.4,
"camera-device-id": 0
"brightness-addition": 30,
"contrast-multiplication": 1.6,
"camera-device-id": 0,
"sharpen": true,
"brighten": false,
"contour-color-decodable": [255, 255, 0],
"contour-color-undecodable": [255, 0, 255]
},
"api":
@ -79,7 +83,7 @@
"open-food-enabled": true,
"open-products-enabled": true,
"best-buy-enabled": true,
"google-books-enabled": false
"google-books-enabled": true
},
"pudding":

1
index.html
View File

@ -121,6 +121,7 @@
/* Flag the game object that new data is available */
Module.flag_frame();
}
}
catch (err)

160
src/Pudding.cpp
View File

@ -996,7 +996,6 @@ void Pudding::capture_frame()
/* Emscripten builds will call this function from the main thread, so don't run continuously */
#ifndef __EMSCRIPTEN__
/* When the camera button is switched off, this thread will automatically finish execution. */
while (camera_switch)
{
@ -1004,40 +1003,56 @@ void Pudding::capture_frame()
* this function, so even if there is data stored there that hasn't been read yet, it should not
* be read by the main thread until this is set to `true`at the end of the function. */
new_frame_available = false;
#endif
/* Load camera frame data into `cv::Mat` */
capture >> camera_frame;
/* If requests are running, a barcode is currently being scanned and the camera image doesn't need to be scanned. */
if (requests.empty())
{
/* Different methods of reading frames depending on the build */
#ifndef __EMSCRIPTEN__
/* Load camera frame data into `cv::Mat` */
capture >> camera_frame;
#else
/* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and read the data
* into a cv::Mat. */
std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
camera_frame = cv::Mat(320, 240, CV_8UC4, emscripten_camera_pixels);
/* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and read the data
* into a cv::Mat. */
std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
#endif
if (!camera_frame.empty())
{
/* Sharpen image for barcode detection */
float sigma = 1.0f, threshold = 5.0f, amount = 1.0f;
cv::GaussianBlur(camera_frame, blurred, cv::Size(), sigma, sigma);
low_contrast_mask = cv::abs(camera_frame - blurred) < threshold;
sharpened_frame = camera_frame * (1.0f + amount) + blurred * (-amount);
camera_frame.copyTo(sharpened_frame, low_contrast_mask);
/* Brightness and contrast adjustment, see https://docs.opencv.org/2.4.13.7/doc/tutorials/core/basic_linear_transform/basic_linear_transform.html */
int brightness = configuration()["scan"]["brightness-addition"];
float contrast = configuration()["scan"]["contrast-multiplication"];
if (brightness != 0 || contrast != 1.0)
if (!camera_frame.empty())
{
camera_frame.convertTo(contrasted_frame, -1, contrast, brightness);
}
if (configuration()["scan"]["sharpen"])
{
/* Sharpen image for barcode detection */
float sigma = 1.0f, threshold = 5.0f, amount = 1.0f;
cv::GaussianBlur(camera_frame, blurred, cv::Size(), sigma, sigma);
low_contrast_mask = cv::abs(camera_frame - blurred) < threshold;
camera_frame = camera_frame * (1.0f + amount) + blurred * (-amount);
camera_frame.copyTo(camera_frame, low_contrast_mask);
}
if (configuration()["scan"]["brighten"])
{
/* Brightness and contrast adjustment, see
* https://docs.opencv.org/2.4.13.7/doc/tutorials/core/basic_linear_transform/basic_linear_transform.html */
int brightness = configuration()["scan"]["brightness-addition"];
float contrast = configuration()["scan"]["contrast-multiplication"];
if (brightness != 0 || contrast != 1.0)
{
camera_frame.convertTo(camera_frame, -1, contrast, brightness);
}
}
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
new_frame_available = true;
}
sb::Log::gl_errors("in capture, after capturing frame");
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
new_frame_available = true;
}
sb::Log::gl_errors("in capture, after capturing frame");
#ifndef __EMSCRIPTEN__
std::this_thread::sleep_for(std::chrono::milliseconds(50));
@ -1073,52 +1088,85 @@ void Pudding::update()
/* Pixels from Emscripten are RGBA */
GLenum pixel_format = GL_RGBA;
/* The cv::Mat rows vs. cols (width vs. height) are correct in Emscripten? */
int camera_frame_width = camera_frame.size[0];
int camera_frame_height = camera_frame.size[1];
#else
/* Pixels from cv::VideoCapture are BGR */
GLenum pixel_format = GL_BGR;
/* The cv::Mat rows vs. cols (width vs. height) values are swapped? */
int camera_frame_width = camera_frame.size[1];
int camera_frame_height = camera_frame.size[0];
#endif
camera_view.texture().bind();
/* Check the original camera frame and the pre-processed ones for barcodes */
for (const cv::Mat& frame : {camera_frame, contrasted_frame, sharpened_frame})
if (configuration()["scan"]["enabled"])
{
/* Fill camera view texture memory with last frame's pixels */
camera_view.texture().load(const_cast<cv::Mat&>(frame).ptr(), {camera_frame_width, camera_frame_height}, pixel_format, GL_UNSIGNED_BYTE);
/* Convert to grayscale, for ZBar */
cv::cvtColor(frame, zbar_frame, cv::COLOR_BGR2GRAY);
if (configuration()["scan"]["enabled"])
std::vector<cv::Point> contours;
/* Check the original camera frame and the pre-processed ones for barcodes */
for (const cv::Mat& frame : {camera_frame, contrasted_frame, sharpened_frame})
{
zbar::Image query_image(
camera_frame_width, camera_frame_height, "Y800", static_cast<void*>(zbar_frame.data), camera_frame_width * camera_frame_height);
int result = image_scanner.scan(query_image);
if (result > 0)
/* Pre-processing may be disabled for this frame */
if (!frame.empty())
{
time_it("barcode lookup")([&] {
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
/* Pass to OpenCV barcode module */
barcode_detector->detectAndDecode(frame, barcode_info, barcode_type, barcode_corners);
/* If there are corners detected, a barcode was detected, so draw the border. */
if (!barcode_corners.empty())
{
if (!barcode_info[0].empty())
{
std::cout << "BAR[" << 0 << "] @ " << cv::Mat(barcode_corners).reshape(2, 1) << ": " <<
"TYPE: " << barcode_type[0] << " INFO: " << barcode_info[0] << std::endl;
}
contours = barcode_corners;
/* If there is barcode info and type, there is a code to store */
if (barcode_info.size() && barcode_type[0] != cv::barcode::NONE && barcode_type[0] != cv::barcode::EAN_8)
{
std::ostringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
message << "camera scanned " << barcode_info[0] << " [" << barcode_type[0] << "]" << std::endl;
sb::Log::log(message);
current_camera_barcode = symbol->get_data();
current_barcode = current_camera_barcode;
current_barcode = barcode_info[0];
break;
}
});
/* No need to check the other pre-processed frames */
break;
}
}
}
/* A barcode was detected, so draw the most recently stored corners */
if (!contours.empty())
{
/* Draw the border of the barcode */
std::vector<std::vector<cv::Point>> input;
input.push_back(contours);
/* Indicate whether the barcode is decodable or not using the border color property */
std::vector<std::uint8_t> color_components;
if (barcode_type[0] != cv::barcode::NONE && barcode_type[0] != cv::barcode::EAN_8)
{
color_components = configuration()["scan"]["contour-color-decodable"].get<std::vector<std::uint8_t>>();
}
else
{
color_components = configuration()["scan"]["contour-color-undecodable"].get<std::vector<std::uint8_t>>();
}
cv::Scalar color = cv::Scalar(color_components[0], color_components[1], color_components[2], 255);
if (pixel_format == GL_BGR)
{
std::uint8_t save = color[0];
color[0] = color[2];
color[2] = save;
}
cv::drawContours(camera_frame, input, 0, color, 2);
for (const cv::Point& point : contours)
{
cv::circle(camera_frame, point, 3, color, cv::FILLED, cv::FILLED);
}
query_image.set_data(nullptr, 0);
}
}
/* Fill camera view texture memory */
camera_view.texture().bind();
camera_view.texture().load(const_cast<cv::Mat&>(camera_frame).ptr(), {camera_frame.cols, camera_frame.rows}, pixel_format, GL_UNSIGNED_BYTE);
/* Frame data has been processed, so there is not a new frame available anymore. */
new_frame_available = false;
}

9
src/Pudding.hpp
View File

@ -42,6 +42,7 @@ using namespace emscripten;
#include "glm/gtx/matrix_decompose.hpp"
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/barcode.hpp"
#include "zbar.h"
#include "Game.hpp"
#include "Color.hpp"
@ -421,7 +422,7 @@ private:
/* Member variables */
std::shared_ptr<SDL_Cursor> poke;
std::string current_barcode, previous_barcode, current_config_barcode, current_camera_barcode;
std::string current_barcode, previous_barcode, current_config_barcode;
std::vector<Item> items;
Item incoming_item;
Carousel item_carousel;
@ -445,6 +446,12 @@ private:
Box viewport, main_viewport, pop_up_viewport;
std::vector<Request*> requests;
/* Storage for barcode scanning module */
cv::Ptr<cv::barcode::BarcodeDetector> barcode_detector = cv::makePtr<cv::barcode::BarcodeDetector>();
std::vector<cv::String> barcode_info;
std::vector<cv::barcode::BarcodeType> barcode_type;
std::vector<cv::Point> barcode_corners;
void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f);
void load_gl_context();
void load_tiles();