use a single GL context by moving texture loading out of camera frame thread and use the thread only for reading frame data into cv::Mat
This commit is contained in:
parent
c500cd332e
commit
17dee6174f
|
@ -63,8 +63,8 @@
|
|||
"best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ",
|
||||
"giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70",
|
||||
"nutronix-enabled": false,
|
||||
"edamam-enabled": true,
|
||||
"open-food-enabled": false,
|
||||
"edamam-enabled": false,
|
||||
"open-food-enabled": true,
|
||||
"open-products-enabled": true,
|
||||
"best-buy-enabled": true,
|
||||
"google-books-enabled": true
|
||||
|
|
|
@ -179,13 +179,13 @@ sb::Texture& Background::current()
|
|||
return carousel.current(textures())->second;
|
||||
}
|
||||
|
||||
CameraView::CameraView() : Plane()
|
||||
PlaneDoubleBuffer::PlaneDoubleBuffer() : Plane()
|
||||
{
|
||||
texture(sb::Texture(), "front");
|
||||
texture(sb::Texture(), "back");
|
||||
}
|
||||
|
||||
void CameraView::generate(const glm::vec2& size)
|
||||
void PlaneDoubleBuffer::generate(const glm::vec2& size)
|
||||
{
|
||||
for (sb::Texture* buffer : {&texture("front"), &texture("back")})
|
||||
{
|
||||
|
@ -193,17 +193,17 @@ void CameraView::generate(const glm::vec2& size)
|
|||
}
|
||||
}
|
||||
|
||||
sb::Texture& CameraView::current()
|
||||
sb::Texture& PlaneDoubleBuffer::active()
|
||||
{
|
||||
return swapped ? texture("back") : texture("front");
|
||||
}
|
||||
|
||||
sb::Texture& CameraView::free()
|
||||
sb::Texture& PlaneDoubleBuffer::inactive()
|
||||
{
|
||||
return swapped ? texture("front") : texture("back");
|
||||
}
|
||||
|
||||
void CameraView::swap()
|
||||
void PlaneDoubleBuffer::swap()
|
||||
{
|
||||
swapped = !swapped;
|
||||
}
|
||||
|
|
|
@ -80,8 +80,11 @@ public:
|
|||
|
||||
};
|
||||
|
||||
/* A Plane that contains a Carousel for cycling through active textures. Only one texture is
|
||||
* active at a time, returned by Background::current. Carousel only goes forward, using Background::next. */
|
||||
/*!
|
||||
* A version of `Plane` that contains a `Carousel` for cycling through active `sb::Texture` objects. Only one
|
||||
* `sb::Texture` is * active at a time, returned by `Background::current`. Carousel only goes forward, using
|
||||
* `Background::next`.
|
||||
*/
|
||||
class Background : public Plane
|
||||
{
|
||||
|
||||
|
@ -96,7 +99,12 @@ public:
|
|||
|
||||
};
|
||||
|
||||
class CameraView : public Plane
|
||||
/*!
|
||||
* A version of `Plane` which contains two texture objects, one of which is active at a time. A reference
|
||||
* to the active `sb::Texture` object is available from `PlaneDoubleBuffer.active`, and the inactive object is
|
||||
* available from `PlaneDoubleBuffer.inactive`. The buffers can be swapped using `PlaneDoubleBuffer.swap`.
|
||||
*/
|
||||
class PlaneDoubleBuffer : public Plane
|
||||
{
|
||||
|
||||
private:
|
||||
|
@ -105,10 +113,10 @@ private:
|
|||
|
||||
public:
|
||||
|
||||
CameraView();
|
||||
PlaneDoubleBuffer();
|
||||
void generate(const glm::vec2&);
|
||||
sb::Texture& current();
|
||||
sb::Texture& free();
|
||||
sb::Texture& active();
|
||||
sb::Texture& inactive();
|
||||
void swap();
|
||||
|
||||
};
|
||||
|
|
200
src/Pudding.cpp
200
src/Pudding.cpp
|
@ -1,11 +1,11 @@
|
|||
/* _______________ ,-------------------------------------------------------------------.
|
||||
//`````````````\\ \ \
|
||||
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
|
||||
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
|
||||
// \\ \ \
|
||||
// \\ \ code released under zlib license [git.nugget.fun/nugget/gunkiss] \
|
||||
// ☆ GUNKISS ☆ \\ \ \
|
||||
//_________________________\\ `-------------------------------------------------------------------'
|
||||
/* _______________ ,--------------------------------------------------------.
|
||||
//`````````````\\ \ \
|
||||
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
|
||||
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
|
||||
// \\ \ \
|
||||
// \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
|
||||
// ☆ GUNKISS ☆ \\ \ \
|
||||
//_________________________\\ `--------------------------------------------------------'
|
||||
|
||||
Generate a custom pudding from food product UPC codes and help a pair of rats take over the video game industry, using
|
||||
their extraterrestrial ability to turn trash into performance enhancing drug puddings that enable business professionals
|
||||
|
@ -31,16 +31,27 @@ Pudding::Pudding()
|
|||
get_delegate().subscribe(&Pudding::respond, this);
|
||||
get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEMOTION);
|
||||
get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEBUTTONDOWN);
|
||||
|
||||
/* initialize a zbar image scanner for reading barcodes of any format */
|
||||
image_scanner.set_config(zbar::ZBAR_NONE, zbar::ZBAR_CFG_ENABLE, 1);
|
||||
|
||||
/* Add a texture to the camera Plane for storing frame image data */
|
||||
camera_view.texture(sb::Texture());
|
||||
|
||||
/* set up pudding model */
|
||||
nlohmann::json pudding = get_configuration()["pudding"];
|
||||
load_pudding_model(pudding["top-radius"], pudding["base-radius"], pudding["ring-vertex-count"], pudding["layer-count"],
|
||||
pudding["y-range"][0], pudding["y-range"][1], pudding["gradient-position"]);
|
||||
|
||||
/* loading GL context instead of SDL context for 3D */
|
||||
load_gl_context();
|
||||
|
||||
/* Load background tiles */
|
||||
load_tiles();
|
||||
|
||||
/* Load button graphics and create button objects */
|
||||
load_pads();
|
||||
|
||||
/* Load a pointer cursor from the system library that will be freed automatically */
|
||||
poke = std::shared_ptr<SDL_Cursor>(SDL_CreateSystemCursor(SDL_SYSTEM_CURSOR_HAND), SDL_FreeCursor);
|
||||
}
|
||||
|
@ -160,12 +171,6 @@ void Pudding::load_gl_context()
|
|||
{
|
||||
super::load_gl_context();
|
||||
|
||||
/* Create another GL context for loading camera frame textures */
|
||||
if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr)
|
||||
{
|
||||
sb::Log::log("could not create capture frame thread context");
|
||||
}
|
||||
|
||||
/* Generate a vertex array object ID, bind it as current (requirement of OpenGL) */
|
||||
vao.generate();
|
||||
vao.bind();
|
||||
|
@ -265,13 +270,14 @@ void Pudding::load_pads()
|
|||
next_button.rotation(glm::radians(180.0f));
|
||||
}
|
||||
|
||||
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage
|
||||
* for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they
|
||||
* can be used as a double buffer.
|
||||
/*!
|
||||
* Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
|
||||
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
|
||||
* a thread which will continuously read frame data.
|
||||
*/
|
||||
void Pudding::initialize_camera()
|
||||
{
|
||||
/* initialize an opencv capture device for getting images from an attached camera */
|
||||
/* Open the OpenCV capture and assign device ID 0 for getting images from the default attached camera. */
|
||||
int device_id = 0;
|
||||
capture.open(device_id);
|
||||
std::ostringstream message;
|
||||
|
@ -280,9 +286,13 @@ void Pudding::initialize_camera()
|
|||
message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" <<
|
||||
capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) <<
|
||||
"fps video capture device ID #" << device_id << " using " << capture.getBackendName();
|
||||
/* generate two textures that will store the video frames with the intention of double buffering them
|
||||
* for threaded texture loading */
|
||||
camera_view.generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)});
|
||||
|
||||
/* Use the texture object to generate a texture the size of the camera's resolution. */
|
||||
camera_view.texture().generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)});
|
||||
|
||||
/* Create and detach a thread which will read frame data */
|
||||
std::thread camera_thread(&Pudding::capture_frame, this);
|
||||
camera_thread.detach();
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -818,100 +828,48 @@ bool Pudding::item_display_active() const
|
|||
return show_item && items.size() > 0;
|
||||
}
|
||||
|
||||
/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread,
|
||||
* so it will use its own GL context to load the pixels into either the front or back texture buffer, depending
|
||||
* on which is not currently in use */
|
||||
int Pudding::capture_frame(void* game)
|
||||
/*!
|
||||
* Read pixels from the camera into a `cv::Mat`. This function is meant to be launched in a separate thread,
|
||||
* where it will run continuously, setting `new_frame_available` to `false` before loading camera frame data
|
||||
* into the `cv::Mat` object at `camera_frame`, then setting `finished_loading_frame` to `true` to indicate
|
||||
* new frame data is available in `camera_frame`.
|
||||
*/
|
||||
void Pudding::capture_frame()
|
||||
{
|
||||
time_it("total thread")([&]{
|
||||
Pudding* pudding = reinterpret_cast<Pudding*>(game);
|
||||
/* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict
|
||||
* with the main rendering context) */
|
||||
if (time_it<int>("make current")([&] { return SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context); }) < 0)
|
||||
/* When capture is closed, this thread will automatically finish execution. */
|
||||
while (capture.isOpened())
|
||||
{
|
||||
sb::Log::log("error making thread context current");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (pudding->capture.isOpened())
|
||||
/* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
|
||||
* this function, so even if there is data stored there that hasn't been read yet, it should not
|
||||
* be read by the main thread until this is set to `true`at the end of the function. */
|
||||
new_frame_available = false;
|
||||
|
||||
/* Load camera frame data into `cv::Mat` */
|
||||
time_it("read frame")([&]{
|
||||
capture >> camera_frame;
|
||||
});
|
||||
|
||||
if (!camera_frame.empty())
|
||||
{
|
||||
cv::Mat frame;
|
||||
time_it("read frame")([&]{
|
||||
pudding->capture.read(frame);
|
||||
/* Rotate the frame 180 degrees to work with OpenGL coords */
|
||||
time_it("flip")([&]{
|
||||
cv::flip(camera_frame, camera_frame, -1);
|
||||
});
|
||||
if (!frame.empty())
|
||||
{
|
||||
time_it("flip")([&]{
|
||||
/* rotate the opencv matrix 180 to work with opengl coords */
|
||||
cv::flip(frame, frame, -1);
|
||||
});
|
||||
time_it("load texture")([&]{
|
||||
/* use whichever texture ID is not being used by the main rendering thread */
|
||||
sb::Texture& texture = pudding->camera_view.free();
|
||||
/* bind texture for accepting pixel data */
|
||||
texture.bind();
|
||||
/* fill texture memory with last frame's pixels */
|
||||
texture.load(frame.ptr(), {frame.cols, frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
|
||||
});
|
||||
pudding->camera_view.swap();
|
||||
if (pudding->get_configuration()["scan"]["enabled"])
|
||||
{
|
||||
time_it("gray")([&]{
|
||||
/* convert to gray and scan with zbar */
|
||||
cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY);
|
||||
});
|
||||
zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast<void*>(frame.data), frame.cols * frame.rows);
|
||||
int result = pudding->image_scanner.scan(query_image);
|
||||
if (result > 0)
|
||||
{
|
||||
time_it("barcode lookup")([&] {
|
||||
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
|
||||
{
|
||||
std::ostringstream message;
|
||||
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
|
||||
sb::Log::log(message);
|
||||
pudding->current_camera_barcode = symbol->get_data();
|
||||
pudding->current_barcode = pudding->current_camera_barcode;
|
||||
}
|
||||
});
|
||||
}
|
||||
query_image.set_data(nullptr, 0);
|
||||
}
|
||||
}
|
||||
frame.release();
|
||||
|
||||
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
|
||||
new_frame_available = true;
|
||||
}
|
||||
SDL_GL_MakeCurrent(pudding->window, nullptr);
|
||||
sb::Log::gl_errors("in capture thread, after capturing frame");
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(50));
|
||||
}
|
||||
pudding->reading_capture_frame = false;
|
||||
// using namespace std::chrono_literals;
|
||||
// std::this_thread::sleep_for(2s);
|
||||
});
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Update parameters and draw the screen */
|
||||
void Pudding::update()
|
||||
{
|
||||
/* number of seconds we've been running for */
|
||||
/* Time in seconds the game has running for */
|
||||
float time_seconds = SDL_GetTicks() / 1000.0f;
|
||||
{
|
||||
/* launch the camera capture thread if it is not currently running */
|
||||
if (capture.isOpened() && !reading_capture_frame)
|
||||
{
|
||||
SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast<void*>(this));
|
||||
if (capture_thread == nullptr)
|
||||
{
|
||||
sb::Log::log("could not create capture thread");
|
||||
}
|
||||
else
|
||||
{
|
||||
reading_capture_frame = true;
|
||||
SDL_WaitThread(capture_thread, nullptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
sb::Log::gl_errors("in main thread, after capturing frame");
|
||||
|
||||
/* if the config is set to refresh automatically, there may be a new barcode available */
|
||||
if (current_config_barcode != get_configuration()["scan"]["barcode"])
|
||||
{
|
||||
|
@ -921,13 +879,49 @@ void Pudding::update()
|
|||
message << "read new barcode from config " << current_barcode;
|
||||
sb::Log::log(message);
|
||||
}
|
||||
|
||||
/* If a new frame is finished loading in the detached thread, copy the frame data into a texture, process the frame
|
||||
* for scanning and scan it. */
|
||||
if (new_frame_available)
|
||||
{
|
||||
camera_view.texture().bind();
|
||||
/* Fill camera view texture memory with last frame's pixels */
|
||||
camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
|
||||
/* Frame data has been loaded, so there is not a new frame available anymore. */
|
||||
new_frame_available = false;
|
||||
/* Convert to grayscale for ZBar */
|
||||
cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY);
|
||||
if (get_configuration()["scan"]["enabled"])
|
||||
{
|
||||
zbar::Image query_image(camera_frame.cols, camera_frame.rows, "Y800", static_cast<void*>(camera_frame.data),
|
||||
camera_frame.cols * camera_frame.rows);
|
||||
int result = image_scanner.scan(query_image);
|
||||
if (result > 0)
|
||||
{
|
||||
time_it("barcode lookup")([&] {
|
||||
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
|
||||
{
|
||||
std::ostringstream message;
|
||||
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
|
||||
sb::Log::log(message);
|
||||
current_camera_barcode = symbol->get_data();
|
||||
current_barcode = current_camera_barcode;
|
||||
}
|
||||
});
|
||||
}
|
||||
query_image.set_data(nullptr, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/* viewport box will be used to tell GL where to draw */
|
||||
viewport = window_box(true);
|
||||
|
||||
/* shrink viewport if item texture or camera will be displayed */
|
||||
if (item_display_active() || capture.isOpened())
|
||||
{
|
||||
viewport.drag_bottom(0.5f * get_configuration()["interface"]["pop-up-viewport-height"].get<float>() * viewport.height());
|
||||
}
|
||||
|
||||
/* Save the main viewport dimensions */
|
||||
main_viewport = viewport;
|
||||
glViewport(viewport);
|
||||
|
@ -1038,7 +1032,7 @@ void Pudding::update()
|
|||
glViewport(viewport);
|
||||
/* bind texture for drawing */
|
||||
glUniformMatrix4fv(uniform["flat"]["transformation"], 1, GL_FALSE, &camera_view.transformation()[0][0]);
|
||||
camera_view.current().bind();
|
||||
camera_view.texture().bind();
|
||||
camera_view.enable();
|
||||
/* draws rectangle vertices and rectangle texture using UV coords */
|
||||
glDrawArrays(GL_TRIANGLES, 0, camera_view.attributes("position")->count());
|
||||
|
|
|
@ -345,23 +345,21 @@ private:
|
|||
Carousel item_carousel;
|
||||
int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
|
||||
cv::VideoCapture capture;
|
||||
cv::Mat camera_frame;
|
||||
zbar::ImageScanner image_scanner;
|
||||
std::map<std::string, std::map<std::string, GLuint>> uniform;
|
||||
GLuint flat_program, mvp_program;
|
||||
glm::mat4 projection, model {1.0f}, mvp;
|
||||
Model pudding_model;
|
||||
Plane plane;
|
||||
Plane plane, camera_view;
|
||||
Background background;
|
||||
CameraView camera_view;
|
||||
bool show_item = false, reading_capture_frame = false;
|
||||
SDL_GLContext capture_frame_thread_context = nullptr;
|
||||
bool show_item = false, new_frame_available = false;
|
||||
sb::VAO vao;
|
||||
sb::VBO vbo;
|
||||
std::map<std::string, sb::Texture> labels;
|
||||
Pad camera_button, previous_button, next_button, inventory_button;
|
||||
Box viewport, main_viewport, pop_up_viewport;
|
||||
std::mutex camera_mutex;
|
||||
SDL_Thread* capture_thread = nullptr;
|
||||
|
||||
void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f);
|
||||
void load_gl_context();
|
||||
|
@ -380,7 +378,7 @@ private:
|
|||
sb::Texture texture_from_image_url(const std::string&) const;
|
||||
static void destroy_texture(GLuint*);
|
||||
bool item_display_active() const;
|
||||
static int capture_frame(void*);
|
||||
void capture_frame();
|
||||
|
||||
/* Initialize camera on connection and release on disconnection. */
|
||||
Connection<> camera_switch {
|
||||
|
|
Loading…
Reference in New Issue