io_pipeline.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489
  1. #include "io_pipeline.h"
  2. #include "camera.h"
  3. #include "flash.h"
  4. #include "main.h"
  5. #include "pipeline.h"
  6. #include "process_pipeline.h"
  7. #include "state.h"
  8. #include <assert.h>
  9. #include <errno.h>
  10. #include <fcntl.h>
  11. #include <glib.h>
  12. #include <math.h>
  13. #include <stdio.h>
  14. #include <string.h>
  15. #include <sys/ioctl.h>
  16. mp_state_io state_io;
  17. MPCamera *mpcamera = NULL;
  18. static MPPipeline *pipeline;
  19. static GSource *capture_source;
  20. static void
  21. setup(MPPipeline *pipeline, const void *data)
  22. {
  23. return;
  24. }
  25. void
  26. mp_io_pipeline_start()
  27. {
  28. mp_process_pipeline_start();
  29. pipeline = mp_pipeline_new();
  30. mp_pipeline_invoke(pipeline, setup, NULL, 0);
  31. }
  32. void
  33. mp_io_pipeline_stop()
  34. {
  35. if (capture_source) {
  36. g_source_destroy(capture_source);
  37. }
  38. mp_pipeline_free(pipeline);
  39. mp_process_pipeline_stop();
  40. }
  41. /*
  42. * Update state from IO -> Process
  43. */
  44. static void
  45. update_process_pipeline()
  46. {
  47. // Grab the latest control values
  48. if (!state_io.gain.manual && state_io.gain.control) {
  49. state_io.gain.value = mp_camera_control_get_int32(
  50. state_io.camera, state_io.gain.control);
  51. }
  52. if (!state_io.exposure.manual && state_io.exposure.control) {
  53. state_io.exposure.value = mp_camera_control_get_int32(
  54. state_io.camera, state_io.exposure.control);
  55. }
  56. float balance_red = 1.0f;
  57. float balance_blue = 1.0f;
  58. if (state_io.red.control && state_io.blue.control) {
  59. int red = mp_camera_control_get_int32(state_io.camera,
  60. state_io.red.control);
  61. int blue = mp_camera_control_get_int32(state_io.camera,
  62. state_io.blue.control);
  63. balance_red = (float)red / (float)state_io.red.max;
  64. balance_blue = (float)blue / (float)state_io.blue.max;
  65. }
  66. struct mp_process_pipeline_state pipeline_state = {
  67. .camera = state_io.camera,
  68. .configuration = state_io.configuration,
  69. .burst_length = state_io.burst_length,
  70. .preview_width = state_io.preview_width,
  71. .preview_height = state_io.preview_height,
  72. .device_rotation = state_io.device_rotation,
  73. .gain_is_manual = state_io.gain.manual,
  74. .gain = state_io.gain.value,
  75. .gain_max = state_io.gain.max,
  76. .balance_red = balance_red,
  77. .balance_blue = balance_blue,
  78. .exposure_is_manual = state_io.exposure.manual,
  79. .exposure = state_io.exposure.value,
  80. .has_auto_focus_continuous = state_io.focus.control != 0,
  81. .has_auto_focus_start = state_io.can_af_trigger,
  82. .flash_enabled = state_io.flash_enabled,
  83. .control_gain = state_io.gain.control != 0,
  84. .control_exposure = state_io.exposure.control != 0,
  85. .control_focus = state_io.focus.control != 0,
  86. .control_flash = true,
  87. };
  88. mp_process_pipeline_update_state(&pipeline_state);
  89. }
  90. static void
  91. focus(MPPipeline *pipeline, const void *data)
  92. {
  93. state_io.trigger_af = true;
  94. }
  95. void
  96. mp_io_pipeline_focus()
  97. {
  98. mp_pipeline_invoke(pipeline, focus, NULL, 0);
  99. }
  100. static void
  101. capture(MPPipeline *pipeline, const void *data)
  102. {
  103. float gain_norm;
  104. // Disable the autogain/exposure while taking the burst
  105. mp_camera_control_set_int32(state_io.camera, V4L2_CID_AUTOGAIN, 0);
  106. mp_camera_control_set_int32(
  107. state_io.camera, V4L2_CID_EXPOSURE_AUTO, V4L2_EXPOSURE_MANUAL);
  108. // Get current gain to calculate a burst length;
  109. // with low gain there's 3, with the max automatic gain of the ov5640
  110. // the value seems to be 248 which creates a 5 frame burst
  111. // for manual gain you can go up to 11 frames
  112. state_io.gain.value =
  113. mp_camera_control_get_int32(state_io.camera, V4L2_CID_GAIN);
  114. gain_norm = (float)state_io.gain.value / (float)state_io.gain.max;
  115. state_io.burst_length = (int)fmax(sqrtf(gain_norm) * 10, 2) + 1;
  116. state_io.burst_length = MAX(1, state_io.burst_length);
  117. state_io.captures_remaining = state_io.burst_length;
  118. // Change camera mode for capturing
  119. mp_process_pipeline_sync();
  120. mp_camera_stop_capture(mpcamera);
  121. struct v4l2_format format = { 0 };
  122. libmegapixels_select_mode(state_io.camera, state_io.mode_capture, &format);
  123. state_io.flush_pipeline = true;
  124. mp_camera_start_capture(mpcamera);
  125. // Enable flash
  126. /* TODO: implement
  127. if (info->flash && flash_enabled) {
  128. mp_flash_enable(info->flash);
  129. }
  130. */
  131. update_process_pipeline();
  132. mp_process_pipeline_capture();
  133. }
  134. void
  135. mp_io_pipeline_capture()
  136. {
  137. mp_pipeline_invoke(pipeline, capture, NULL, 0);
  138. }
  139. static void
  140. release_buffer(MPPipeline *pipeline, const uint32_t *buffer_index)
  141. {
  142. mp_camera_release_buffer(mpcamera, *buffer_index);
  143. }
  144. void
  145. mp_io_pipeline_release_buffer(uint32_t buffer_index)
  146. {
  147. mp_pipeline_invoke(pipeline,
  148. (MPPipelineCallback)release_buffer,
  149. &buffer_index,
  150. sizeof(uint32_t));
  151. }
  152. static pid_t focus_continuous_task = 0;
  153. static pid_t start_focus_task = 0;
  154. static void
  155. start_focus()
  156. {
  157. // only run 1 manual focus at once
  158. if (!mp_camera_check_task_complete(mpcamera, start_focus_task) ||
  159. !mp_camera_check_task_complete(mpcamera, focus_continuous_task))
  160. return;
  161. if (state_io.focus.control) {
  162. focus_continuous_task = mp_camera_control_set_bool_bg(
  163. state_io.camera, state_io.focus.control, 1);
  164. } else if (state_io.can_af_trigger) {
  165. start_focus_task = mp_camera_control_set_bool_bg(
  166. state_io.camera, V4L2_CID_AUTO_FOCUS_START, 1);
  167. }
  168. }
  169. static void
  170. update_controls()
  171. {
  172. // Don't update controls while capturing
  173. if (state_io.captures_remaining > 0) {
  174. return;
  175. }
  176. if (state_io.trigger_af) {
  177. state_io.trigger_af = false;
  178. start_focus();
  179. }
  180. if (state_io.gain.manual != state_io.gain.manual_req) {
  181. mp_camera_control_set_bool_bg(state_io.camera,
  182. V4L2_CID_AUTOGAIN,
  183. !state_io.gain.manual_req);
  184. state_io.gain.manual = state_io.gain.manual_req;
  185. }
  186. if (state_io.gain.manual && state_io.gain.value != state_io.gain.value_req) {
  187. mp_camera_control_set_int32_bg(state_io.camera,
  188. state_io.gain.control,
  189. state_io.gain.value_req);
  190. state_io.gain.value = state_io.gain.value_req;
  191. }
  192. if (state_io.exposure.manual != state_io.exposure.manual_req) {
  193. mp_camera_control_set_bool_bg(state_io.camera,
  194. V4L2_CID_EXPOSURE_AUTO,
  195. state_io.exposure.manual_req ?
  196. V4L2_EXPOSURE_MANUAL :
  197. V4L2_EXPOSURE_AUTO);
  198. state_io.exposure.manual = state_io.exposure.manual_req;
  199. }
  200. if (state_io.exposure.manual &&
  201. state_io.exposure.value != state_io.exposure.value_req) {
  202. mp_camera_control_set_int32_bg(state_io.camera,
  203. state_io.exposure.control,
  204. state_io.exposure.value_req);
  205. state_io.exposure.value = state_io.exposure.value_req;
  206. }
  207. }
  208. static void
  209. on_frame(MPBuffer buffer, void *_data)
  210. {
  211. // Don't process frame when the window is not active
  212. if (!check_window_active()) {
  213. return;
  214. }
  215. // Only update controls right after a frame was captured
  216. update_controls();
  217. // When the mode is switched while capturing we get a couple blank frames,
  218. // presumably from buffers made ready during the switch. Ignore these.
  219. if (state_io.flush_pipeline) {
  220. if (state_io.blank_frame_count < 20) {
  221. // Only check a 10x10 area
  222. size_t test_size =
  223. MIN(10, state_io.camera->current_mode->width) *
  224. MIN(10, state_io.camera->current_mode->height);
  225. bool image_is_blank = true;
  226. for (size_t i = 0; i < test_size; ++i) {
  227. if (buffer.data[i] != 0) {
  228. image_is_blank = false;
  229. }
  230. }
  231. if (image_is_blank) {
  232. ++state_io.blank_frame_count;
  233. return;
  234. }
  235. } else {
  236. printf("Blank image limit reached, resulting capture may be blank\n");
  237. }
  238. state_io.flush_pipeline = false;
  239. state_io.blank_frame_count = 0;
  240. }
  241. // Send the image off for processing
  242. mp_process_pipeline_process_image(buffer);
  243. if (state_io.captures_remaining > 0) {
  244. --state_io.captures_remaining;
  245. if (state_io.captures_remaining == 0) {
  246. // Restore the auto exposure and gain if needed
  247. if (!state_io.exposure.manual) {
  248. mp_camera_control_set_int32_bg(
  249. state_io.camera,
  250. V4L2_CID_EXPOSURE_AUTO,
  251. V4L2_EXPOSURE_AUTO);
  252. }
  253. if (!state_io.gain.manual) {
  254. mp_camera_control_set_bool_bg(
  255. state_io.camera, V4L2_CID_AUTOGAIN, true);
  256. }
  257. // Go back to preview mode
  258. mp_process_pipeline_sync();
  259. mp_camera_stop_capture(mpcamera);
  260. struct v4l2_format format = { 0 };
  261. libmegapixels_select_mode(
  262. state_io.camera, state_io.mode_preview, &format);
  263. state_io.flush_pipeline = true;
  264. mp_camera_start_capture(mpcamera);
  265. // Disable flash
  266. /* TODO: implement
  267. if (info->flash && flash_enabled) {
  268. mp_flash_disable(info->flash);
  269. }
  270. */
  271. update_process_pipeline();
  272. }
  273. }
  274. }
  275. static void
  276. init_controls()
  277. {
  278. if (mp_camera_query_control(
  279. state_io.camera, V4L2_CID_FOCUS_ABSOLUTE, NULL)) {
  280. // TODO: Set focus state
  281. state_io.focus.control = V4L2_CID_FOCUS_ABSOLUTE;
  282. } else {
  283. state_io.focus.control = 0;
  284. }
  285. if (mp_camera_query_control(state_io.camera, V4L2_CID_FOCUS_AUTO, NULL)) {
  286. mp_camera_control_set_bool_bg(
  287. state_io.camera, V4L2_CID_FOCUS_AUTO, true);
  288. }
  289. state_io.can_af_trigger = mp_camera_query_control(
  290. state_io.camera, V4L2_CID_AUTO_FOCUS_START, NULL);
  291. MPControl control;
  292. if (mp_camera_query_control(state_io.camera, V4L2_CID_GAIN, &control)) {
  293. state_io.gain.control = V4L2_CID_GAIN;
  294. state_io.gain.max = control.max;
  295. } else if (mp_camera_query_control(
  296. state_io.camera, V4L2_CID_ANALOGUE_GAIN, &control)) {
  297. state_io.gain.control = V4L2_CID_ANALOGUE_GAIN;
  298. state_io.gain.max = control.max;
  299. } else {
  300. state_io.gain.max = 0;
  301. state_io.gain.control = 0;
  302. }
  303. if (state_io.gain.control) {
  304. state_io.gain.value = mp_camera_control_get_int32(
  305. state_io.camera, state_io.gain.control);
  306. } else {
  307. state_io.gain.value = 0;
  308. }
  309. state_io.gain.manual =
  310. mp_camera_control_get_bool(state_io.camera, V4L2_CID_AUTOGAIN) == 0;
  311. state_io.exposure.value =
  312. mp_camera_control_get_int32(state_io.camera, V4L2_CID_EXPOSURE);
  313. state_io.exposure.manual =
  314. mp_camera_control_get_int32(state_io.camera,
  315. V4L2_CID_EXPOSURE_AUTO) ==
  316. V4L2_EXPOSURE_MANUAL;
  317. if (mp_camera_query_control(
  318. state_io.camera, V4L2_CID_RED_BALANCE, &control)) {
  319. state_io.red.control = V4L2_CID_RED_BALANCE;
  320. state_io.red.max = control.max;
  321. } else {
  322. state_io.red.control = 0;
  323. }
  324. if (mp_camera_query_control(
  325. state_io.camera, V4L2_CID_BLUE_BALANCE, &control)) {
  326. state_io.blue.control = V4L2_CID_BLUE_BALANCE;
  327. state_io.blue.max = control.max;
  328. } else {
  329. state_io.blue.control = 0;
  330. }
  331. }
  332. /*
  333. * State transfer from Main -> IO
  334. */
  335. static void
  336. update_state(MPPipeline *pipeline, const struct mp_io_pipeline_state *state)
  337. {
  338. if (state_io.camera != state->camera) {
  339. if (state_io.camera != NULL) {
  340. mp_process_pipeline_sync();
  341. mp_camera_stop_capture(mpcamera);
  342. libmegapixels_close(state_io.camera);
  343. }
  344. if (capture_source) {
  345. g_source_destroy(capture_source);
  346. capture_source = NULL;
  347. }
  348. state_io.camera = state->camera;
  349. if (state_io.camera) {
  350. libmegapixels_open(state_io.camera);
  351. mpcamera = mp_camera_new(state_io.camera);
  352. state_io.mode_preview = NULL;
  353. state_io.mode_capture = NULL;
  354. float score = 0;
  355. int area_preview =
  356. state_io.preview_width * state_io.preview_height;
  357. for (int m = 0; m < state_io.camera->num_modes; m++) {
  358. float mscore = 0;
  359. if (state_io.camera->modes[m]->rate > 29) {
  360. mscore += 1;
  361. }
  362. int mode_area = state_io.camera->modes[m]->width *
  363. state_io.camera->modes[m]->height;
  364. mscore += 1.0f -
  365. (float)(ABS(mode_area - area_preview) /
  366. area_preview);
  367. if (mscore > score) {
  368. state_io.mode_preview =
  369. state_io.camera->modes[m];
  370. score = mscore;
  371. }
  372. }
  373. long area = 0;
  374. for (int m = 0; m < state_io.camera->num_modes; m++) {
  375. long this_pixels = state_io.camera->modes[m]->width *
  376. state_io.camera->modes[m]->height;
  377. if (this_pixels > area) {
  378. area = this_pixels;
  379. state_io.mode_capture =
  380. state_io.camera->modes[m];
  381. }
  382. }
  383. if (state_io.mode_preview == NULL &&
  384. state_io.mode_capture != NULL) {
  385. // If no fast preview mode is available, make due
  386. // with slow modes.
  387. state_io.mode_preview = state_io.mode_capture;
  388. }
  389. if (state_io.mode_preview != NULL) {
  390. if (state_io.camera->video_fd == 0) {
  391. libmegapixels_open(state_io.camera);
  392. }
  393. struct v4l2_format format = { 0 };
  394. libmegapixels_select_mode(state_io.camera,
  395. state_io.mode_preview,
  396. &format);
  397. }
  398. mp_camera_start_capture(mpcamera);
  399. capture_source = mp_pipeline_add_capture_source(
  400. pipeline, mpcamera, on_frame, NULL);
  401. init_controls();
  402. }
  403. }
  404. state_io.configuration = state->configuration;
  405. state_io.burst_length = state->burst_length;
  406. state_io.preview_width = state->preview_width;
  407. state_io.preview_height = state->preview_height;
  408. state_io.device_rotation = state->device_rotation;
  409. if (state_io.camera) {
  410. state_io.gain.manual_req = state->gain_is_manual;
  411. state_io.gain.value_req = state->gain;
  412. state_io.exposure.manual_req = state->exposure_is_manual;
  413. state_io.exposure.value_req = state->exposure;
  414. state_io.flash_enabled = state->flash_enabled;
  415. }
  416. update_process_pipeline();
  417. }
  418. void
  419. mp_io_pipeline_update_state(const struct mp_io_pipeline_state *state)
  420. {
  421. mp_pipeline_invoke(pipeline,
  422. (MPPipelineCallback)update_state,
  423. state,
  424. sizeof(struct mp_io_pipeline_state));
  425. }