Skip to content

Commit 890361a

Browse files
gordinmityagordinmitya
authored
yuv2buf for processing ImageReader output (#405)
1 parent 42f76c4 commit 890361a

File tree

2 files changed

+347
-152
lines changed

2 files changed

+347
-152
lines changed

‎CameraUtils/lib/src/main/java/com/example/android/camera/utils/Yuv.java‎

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
package com.example.android.camera.utils;
2+
3+
import android.graphics.ImageFormat;
4+
import android.media.Image;
5+
6+
import androidx.annotation.IntDef;
7+
8+
import java.lang.annotation.Retention;
9+
import java.lang.annotation.RetentionPolicy;
10+
import java.nio.ByteBuffer;
11+
12+
13+
abstract public class Yuv {
14+
/*
15+
This file is part of https://github.com/gordinmitya/yuv2buf.
16+
Follow the link to find demo app, performance benchmarks and unit tests.
17+
18+
Intro to YUV image formats:
19+
YUV_420_888 - is a generic format that can be represented as I420, YV12, NV21, and NV12.
20+
420 means that for each 4 luminosity pixels we have 2 chroma pixels: U and V.
21+
22+
* I420 format represents an image as Y plane followed by U then followed by V plane
23+
without chroma channels interleaving.
24+
For example:
25+
Y Y Y Y
26+
Y Y Y Y
27+
U U V V
28+
29+
* NV21 format represents an image as Y plane followed by V and U interleaved. First V then U.
30+
For example:
31+
Y Y Y Y
32+
Y Y Y Y
33+
V U V U
34+
35+
* YV12 and NV12 are the same as previous formats but with swapped order of V and U. (U then V)
36+
37+
Visualization of these 4 formats:
38+
https://user-images.githubusercontent.com/9286092/89119601-4f6f8100-d4b8-11ea-9a51-2765f7e513c2.jpg
39+
40+
It's guaranteed that image.getPlanes() always returns planes in order Y U V for YUV_420_888.
41+
https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
42+
43+
Because I420 and NV21 are more widely supported (RenderScript, OpenCV, MNN)
44+
the conversion is done into these formats.
45+
46+
More about each format: https://www.fourcc.org/yuv.php
47+
*/
48+
49+
@Retention(RetentionPolicy.SOURCE)
50+
@IntDef({ImageFormat.NV21, ImageFormat.YUV_420_888})
51+
public @interface YuvType {
52+
}
53+
54+
public static class Converted {
55+
@YuvType
56+
public final int type;
57+
public final ByteBuffer buffer;
58+
59+
private Converted(@YuvType int type, ByteBuffer buffer) {
60+
this.type = type;
61+
this.buffer = buffer;
62+
}
63+
}
64+
65+
/*
66+
Api.
67+
*/
68+
@YuvType
69+
public static int detectType(Image image) {
70+
return detectType(wrap(image));
71+
}
72+
73+
public static Converted toBuffer(Image image) {
74+
return toBuffer(image, null);
75+
}
76+
77+
public static Converted toBuffer(Image image, ByteBuffer reuse) {
78+
return toBuffer(wrap(image), reuse);
79+
}
80+
81+
private static ImageWrapper wrap(Image image) {
82+
int width = image.getWidth();
83+
int height = image.getHeight();
84+
Image.Plane[] planes = image.getPlanes();
85+
PlaneWrapper y = wrap(width, height, planes[0]);
86+
PlaneWrapper u = wrap(width / 2, height / 2, planes[1]);
87+
PlaneWrapper v = wrap(width / 2, height / 2, planes[2]);
88+
return new ImageWrapper(width, height, y, u, v);
89+
}
90+
91+
private static PlaneWrapper wrap(int width, int height, Image.Plane plane) {
92+
return new PlaneWrapper(
93+
width,
94+
height,
95+
plane.getBuffer(),
96+
plane.getRowStride(),
97+
plane.getPixelStride()
98+
);
99+
}
100+
101+
/*
102+
CameraX api. If you don't need it – just comment lines below.
103+
*/
104+
/*
105+
@YuvType
106+
public static int detectType(ImageProxy image) {
107+
return detectType(wrap(image));
108+
}
109+
110+
public static Converted toBuffer(ImageProxy image) {
111+
return toBuffer(image, null);
112+
}
113+
114+
public static Converted toBuffer(ImageProxy image, ByteBuffer reuse) {
115+
return toBuffer(wrap(image), reuse);
116+
}
117+
118+
private static ImageWrapper wrap(ImageProxy image) {
119+
int width = image.getWidth();
120+
int height = image.getHeight();
121+
ImageProxy.PlaneProxy[] planes = image.getPlanes();
122+
PlaneWrapper y = wrap(width, height, planes[0]);
123+
PlaneWrapper u = wrap(width / 2, height / 2, planes[1]);
124+
PlaneWrapper v = wrap(width / 2, height / 2, planes[2]);
125+
return new ImageWrapper(width, height, y, u, v);
126+
}
127+
128+
private static PlaneWrapper wrap(int width, int height, ImageProxy.PlaneProxy plane) {
129+
return new PlaneWrapper(
130+
width,
131+
height,
132+
plane.getBuffer(),
133+
plane.getRowStride(),
134+
plane.getPixelStride()
135+
);
136+
}
137+
*/
138+
// End of CameraX api.
139+
140+
/*
141+
Implementation
142+
*/
143+
144+
/*
145+
other pixelStride are not possible
146+
@see #ImageWrapper.checkFormat()
147+
*/
148+
@YuvType
149+
static int detectType(ImageWrapper image) {
150+
if (image.u.pixelStride == 1) {
151+
return ImageFormat.YUV_420_888;
152+
} else {
153+
return ImageFormat.NV21;
154+
}
155+
}
156+
157+
static Converted toBuffer(ImageWrapper image, ByteBuffer reuse) {
158+
final int type = detectType(image);
159+
ByteBuffer output = prepareOutput(image, reuse);
160+
removePadding(image, type, output);
161+
return new Converted(type, output);
162+
}
163+
164+
private static ByteBuffer prepareOutput(ImageWrapper image, ByteBuffer reuse) {
165+
int sizeOutput = image.width * image.height * 3 / 2;
166+
ByteBuffer output;
167+
if (reuse == null
168+
|| reuse.capacity() < sizeOutput
169+
|| reuse.isReadOnly()
170+
|| !reuse.isDirect()) {
171+
output = ByteBuffer.allocateDirect(sizeOutput);
172+
} else {
173+
output = reuse;
174+
}
175+
output.rewind();
176+
return output;
177+
}
178+
179+
// Input buffers are always direct as described in
180+
// https://developer.android.com/reference/android/media/Image.Plane#getBuffer()
181+
private static void removePadding(
182+
ImageWrapper image,
183+
@YuvType final int type,
184+
ByteBuffer output
185+
) {
186+
int sizeLuma = image.y.width * image.y.height;
187+
int sizeChroma = image.u.width * image.u.height;
188+
189+
if (image.y.rowStride > image.y.width) {
190+
removePaddingCompact(image.y, output, 0);
191+
} else {
192+
output.position(0);
193+
output.put(image.y.buffer);
194+
}
195+
196+
if (type == ImageFormat.YUV_420_888) {
197+
if (image.u.rowStride > image.u.width) {
198+
removePaddingCompact(image.u, output, sizeLuma);
199+
removePaddingCompact(image.v, output, sizeLuma + sizeChroma);
200+
} else {
201+
output.position(sizeLuma);
202+
output.put(image.u.buffer);
203+
output.position(sizeLuma + sizeChroma);
204+
output.put(image.v.buffer);
205+
}
206+
} else {
207+
if (image.u.rowStride > image.u.width * 2) {
208+
removePaddingNotCompact(image, output, sizeLuma);
209+
} else {
210+
output.position(sizeLuma);
211+
ByteBuffer uv = image.v.buffer;
212+
final int properUVSize = image.v.height * image.v.rowStride - 1;
213+
if (uv.capacity() > properUVSize) {
214+
uv = clipBuffer(image.v.buffer, 0, properUVSize);
215+
}
216+
output.put(uv);
217+
final byte lastOne = image.u.buffer.get(image.u.buffer.capacity() - 1);
218+
output.put(output.capacity() - 1, lastOne);
219+
}
220+
}
221+
output.rewind();
222+
}
223+
224+
private static void removePaddingCompact(PlaneWrapper plane, ByteBuffer dst, int offset) {
225+
if (plane.pixelStride != 1) {
226+
throw new IllegalArgumentException("use removePaddingCompact with pixelStride == 1");
227+
}
228+
229+
ByteBuffer src = plane.buffer;
230+
int rowStride = plane.rowStride;
231+
ByteBuffer row;
232+
dst.position(offset);
233+
for (int i = 0; i < plane.height; i++) {
234+
row = clipBuffer(src, i * rowStride, plane.width);
235+
dst.put(row);
236+
}
237+
}
238+
239+
private static void removePaddingNotCompact(ImageWrapper image, ByteBuffer dst, int offset) {
240+
if (image.u.pixelStride != 2) {
241+
throw new IllegalArgumentException("use removePaddingNotCompact pixelStride == 2");
242+
}
243+
244+
int width = image.u.width;
245+
int height = image.u.height;
246+
int rowStride = image.u.rowStride;
247+
ByteBuffer row;
248+
dst.position(offset);
249+
for (int i = 0; i < height - 1; i++) {
250+
row = clipBuffer(image.v.buffer, i * rowStride, width * 2);
251+
dst.put(row);
252+
}
253+
row = clipBuffer(image.u.buffer, (height - 1) * rowStride - 1, width * 2);
254+
dst.put(row);
255+
}
256+
257+
private static ByteBuffer clipBuffer(ByteBuffer buffer, int start, int size) {
258+
ByteBuffer duplicate = buffer.duplicate();
259+
duplicate.position(start);
260+
duplicate.limit(start + size);
261+
return duplicate.slice();
262+
}
263+
264+
static class ImageWrapper {
265+
final int width, height;
266+
final PlaneWrapper y, u, v;
267+
268+
ImageWrapper(int width, int height, PlaneWrapper y, PlaneWrapper u, PlaneWrapper v) {
269+
this.width = width;
270+
this.height = height;
271+
this.y = y;
272+
this.u = u;
273+
this.v = v;
274+
checkFormat();
275+
}
276+
277+
278+
// Check this is a supported image format
279+
// https://developer.android.com/reference/android/graphics/ImageFormat#YUV_420_888
280+
private void checkFormat() {
281+
if (y.pixelStride != 1) {
282+
throw new IllegalArgumentException(String.format(
283+
"Pixel stride for Y plane must be 1 but got %d instead",
284+
y.pixelStride
285+
));
286+
}
287+
if (u.pixelStride != v.pixelStride || u.rowStride != v.rowStride) {
288+
throw new IllegalArgumentException(String.format(
289+
"U and V planes must have the same pixel and row strides " +
290+
"but got pixel=%d row=%d for U " +
291+
"and pixel=%d and row=%d for V",
292+
u.pixelStride, u.rowStride,
293+
v.pixelStride, v.rowStride
294+
));
295+
}
296+
if (u.pixelStride != 1 && u.pixelStride != 2) {
297+
throw new IllegalArgumentException(
298+
"Supported pixel strides for U and V planes are 1 and 2"
299+
);
300+
}
301+
}
302+
}
303+
304+
static class PlaneWrapper {
305+
final int width, height;
306+
final ByteBuffer buffer;
307+
final int rowStride, pixelStride;
308+
309+
PlaneWrapper(int width, int height, ByteBuffer buffer, int rowStride, int pixelStride) {
310+
this.width = width;
311+
this.height = height;
312+
this.buffer = buffer;
313+
this.rowStride = rowStride;
314+
this.pixelStride = pixelStride;
315+
}
316+
}
317+
}

0 commit comments

Comments
 (0)