FFmpeg: libavcodec/vmnc.c Source File

FFmpeg

[フレーム]

libavcodec

vmnc.c

Go to the documentation of this file.

1 /*

2 * VMware Screen Codec (VMnc) decoder

4 *

5 * This file is part of FFmpeg.

6 *

7 * FFmpeg is free software; you can redistribute it and/or

8 * modify it under the terms of the GNU Lesser General Public

9 * License as published by the Free Software Foundation; either

10 * version 2.1 of the License, or (at your option) any later version.

11 *

12 * FFmpeg is distributed in the hope that it will be useful,

13 * but WITHOUT ANY WARRANTY; without even the implied warranty of

14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

15 * Lesser General Public License for more details.

16 *

17 * You should have received a copy of the GNU Lesser General Public

18 * License along with FFmpeg; if not, write to the Free Software

19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

20 */

22 /**

23 * @file

24 * VMware Screen Codec (VMnc) decoder

25 * As Alex Beregszaszi discovered, this is effectively RFB data dump

26 */

28 #include <stdio.h>

29 #include <stdlib.h>

31 #include "libavutil/common.h"

32 #include "libavutil/intreadwrite.h"

33 #include "avcodec.h"

34 #include "internal.h"

35 #include "bytestream.h"

37 enum EncTypes {

38 MAGIC_WMVd = 0x574D5664,

39 MAGIC_WMVe,

40 MAGIC_WMVf,

41 MAGIC_WMVg,

42 MAGIC_WMVh,

43 MAGIC_WMVi,

44 MAGIC_WMVj

45 };

47 enum HexTile_Flags {

48 HT_RAW = 1, // tile is raw

49 HT_BKG = 2, // background color is present

50 HT_FG = 4, // foreground color is present

51 HT_SUB = 8, // subrects are present

52 HT_CLR = 16 // each subrect has own color

53 };

55 /*

56 * Decoder context

57 */

58 typedef struct VmncContext {

59 AVCodecContext *avctx;

60 AVFrame *pic;

62 int bpp;

63 int bpp2;

64 int bigendian;

65 uint8_t pal[768];

66 int width, height;

67 GetByteContext gb;

69 /* cursor data */

70 int cur_w, cur_h;

71 int cur_x, cur_y;

72 int cur_hx, cur_hy;

73 uint8_t *curbits, *curmask;

74 uint8_t *screendta;

75 } VmncContext;

77 /* read pixel value from stream */

78 static av_always_inline int vmnc_get_pixel(GetByteContext *gb, int bpp, int be)

79 {

80 switch (bpp * 2 + be) {

81 case 2:

82 case 3:

83 return bytestream2_get_byte(gb);

84 case 4:

85 return bytestream2_get_le16(gb);

86 case 5:

87 return bytestream2_get_be16(gb);

88 case 8:

89 return bytestream2_get_le32(gb);

90 case 9:

91 return bytestream2_get_be32(gb);

92 default: return 0;

93 }

94 }

96 static void load_cursor(VmncContext *c)

97 {

98 int i, j, p;

99 const int bpp = c->bpp2;

100 uint8_t *dst8 = c->curbits;

101 uint16_t *dst16 = (uint16_t *)c->curbits;

102 uint32_t *dst32 = (uint32_t *)c->curbits;

103

104 for (j = 0; j < c->cur_h; j++) {

105 for (i = 0; i < c->cur_w; i++) {

106 p = vmnc_get_pixel(&c->gb, bpp, c->bigendian);

107 if (bpp == 1)

108 *dst8++ = p;

109 if (bpp == 2)

110 *dst16++ = p;

111 if (bpp == 4)

112 *dst32++ = p;

113 }

114 }

115 dst8 = c->curmask;

116 dst16 = (uint16_t*)c->curmask;

117 dst32 = (uint32_t*)c->curmask;

118 for (j = 0; j < c->cur_h; j++) {

119 for (i = 0; i < c->cur_w; i++) {

120 p = vmnc_get_pixel(&c->gb, bpp, c->bigendian);

121 if (bpp == 1)

122 *dst8++ = p;

123 if (bpp == 2)

124 *dst16++ = p;

125 if (bpp == 4)

126 *dst32++ = p;

127 }

128 }

129 }

130

131 static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)

132 {

133 int i, j;

134 int w, h, x, y;

135 w = c->cur_w;

136 if (c->width < c->cur_x + c->cur_w)

137 w = c->width - c->cur_x;

138 h = c->cur_h;

139 if (c->height < c->cur_y + c->cur_h)

140 h = c->height - c->cur_y;

141 x = c->cur_x;

142 y = c->cur_y;

143 if (x < 0) {

144 w += x;

145 x = 0;

146 }

147 if (y < 0) {

148 h += y;

149 y = 0;

150 }

151

152 if ((w < 1) || (h < 1))

153 return;

154 dst += x * c->bpp2 + y * stride;

155

156 if (c->bpp2 == 1) {

157 uint8_t *cd = c->curbits, *msk = c->curmask;

158 for (j = 0; j < h; j++) {

159 for (i = 0; i < w; i++)

160 dst[i] = (dst[i] & cd[i]) ^ msk[i];

161 msk += c->cur_w;

162 cd += c->cur_w;

163 dst += stride;

164 }

165 } else if (c->bpp2 == 2) {

166 uint16_t *cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;

167 uint16_t *dst2;

168 for (j = 0; j < h; j++) {

169 dst2 = (uint16_t*)dst;

170 for (i = 0; i < w; i++)

171 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];

172 msk += c->cur_w;

173 cd += c->cur_w;

174 dst += stride;

175 }

176 } else if (c->bpp2 == 4) {

177 uint32_t *cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;

178 uint32_t *dst2;

179 for (j = 0; j < h; j++) {

180 dst2 = (uint32_t*)dst;

181 for (i = 0; i < w; i++)

182 dst2[i] = (dst2[i] & cd[i]) ^ msk[i];

183 msk += c->cur_w;

184 cd += c->cur_w;

185 dst += stride;

186 }

187 }

188 }

189

190 /* fill rectangle with given color */

191 static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy,

192 int w, int h, int color,

193 int bpp, int stride)

194 {

195 int i, j;

196 dst += dx * bpp + dy * stride;

197 if (bpp == 1) {

198 for (j = 0; j < h; j++) {

199 memset(dst, color, w);

200 dst += stride;

201 }

202 } else if (bpp == 2) {

203 uint16_t *dst2;

204 for (j = 0; j < h; j++) {

205 dst2 = (uint16_t*)dst;

206 for (i = 0; i < w; i++)

207 *dst2++ = color;

208 dst += stride;

209 }

210 } else if (bpp == 4) {

211 uint32_t *dst2;

212 for (j = 0; j < h; j++) {

213 dst2 = (uint32_t*)dst;

214 for (i = 0; i < w; i++)

215 dst2[i] = color;

216 dst += stride;

217 }

218 }

219 }

220

221 static av_always_inline void paint_raw(uint8_t *dst, int w, int h,

222 GetByteContext *gb, int bpp,

223 int be, int stride)

224 {

225 int i, j, p;

226 for (j = 0; j < h; j++) {

227 for (i = 0; i < w; i++) {

228 p = vmnc_get_pixel(gb, bpp, be);

229 switch (bpp) {

230 case 1:

231 dst[i] = p;

232 break;

233 case 2:

234 ((uint16_t*)dst)[i] = p;

235 break;

236 case 4:

237 ((uint32_t*)dst)[i] = p;

238 break;

239 }

240 }

241 dst += stride;

242 }

243 }

244

245 static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb,

246 int w, int h, int stride)

247 {

248 int i, j, k;

249 int bg = 0, fg = 0, rects, color, flags, xy, wh;

250 const int bpp = c->bpp2;

251 uint8_t *dst2;

252 int bw = 16, bh = 16;

253

254 for (j = 0; j < h; j += 16) {

255 dst2 = dst;

256 bw = 16;

257 if (j + 16 > h)

258 bh = h - j;

259 for (i = 0; i < w; i += 16, dst2 += 16 * bpp) {

260 if (bytestream2_get_bytes_left(gb) <= 0) {

261 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");

262 return AVERROR_INVALIDDATA;

263 }

264 if (i + 16 > w)

265 bw = w - i;

266 flags = bytestream2_get_byte(gb);

267 if (flags & HT_RAW) {

268 if (bytestream2_get_bytes_left(gb) < bw * bh * bpp) {

269 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");

270 return AVERROR_INVALIDDATA;

271 }

272 paint_raw(dst2, bw, bh, gb, bpp, c->bigendian, stride);

273 } else {

274 if (flags & HT_BKG)

275 bg = vmnc_get_pixel(gb, bpp, c->bigendian);

276 if (flags & HT_FG)

277 fg = vmnc_get_pixel(gb, bpp, c->bigendian);

278 rects = 0;

279 if (flags & HT_SUB)

280 rects = bytestream2_get_byte(gb);

281 color = !!(flags & HT_CLR);

282

283 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);

284

285 if (bytestream2_get_bytes_left(gb) < rects * (color * bpp + 2)) {

286 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");

287 return AVERROR_INVALIDDATA;

288 }

289 for (k = 0; k < rects; k++) {

290 if (color)

291 fg = vmnc_get_pixel(gb, bpp, c->bigendian);

292 xy = bytestream2_get_byte(gb);

293 wh = bytestream2_get_byte(gb);

294 if ( (xy >> 4) + (wh >> 4) + 1 > w - i

295 || (xy & 0xF) + (wh & 0xF)+1 > h - j) {

296 av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n");

297 return AVERROR_INVALIDDATA;

298 }

299 paint_rect(dst2, xy >> 4, xy & 0xF,

300 (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);

301 }

302 }

303 }

304 dst += stride * 16;

305 }

306 return 0;

307 }

308

309 static void reset_buffers(VmncContext *c)

310 {

311 av_freep(&c->curbits);

312 av_freep(&c->curmask);

313 av_freep(&c->screendta);

314 c->cur_w = c->cur_h = 0;

315 c->cur_hx = c->cur_hy = 0;

316

317 }

318

319 static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,

320 AVPacket *avpkt)

321 {

322 const uint8_t *buf = avpkt->data;

323 int buf_size = avpkt->size;

324 VmncContext * const c = avctx->priv_data;

325 GetByteContext *gb = &c->gb;

326 uint8_t *outptr;

327 int dx, dy, w, h, depth, enc, chunks, res, size_left, ret;

328

329 if ((ret = ff_reget_buffer(avctx, c->pic)) < 0)

330 return ret;

331

332 bytestream2_init(gb, buf, buf_size);

333

334 c->pic->key_frame = 0;

335 c->pic->pict_type = AV_PICTURE_TYPE_P;

336

337 // restore screen after cursor

338 if (c->screendta) {

339 int i;

340 w = c->cur_w;

341 if (c->width < c->cur_x + w)

342 w = c->width - c->cur_x;

343 h = c->cur_h;

344 if (c->height < c->cur_y + h)

345 h = c->height - c->cur_y;

346 dx = c->cur_x;

347 if (dx < 0) {

348 w += dx;

349 dx = 0;

350 }

351 dy = c->cur_y;

352 if (dy < 0) {

353 h += dy;

354 dy = 0;

355 }

356 if ((w > 0) && (h > 0)) {

357 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];

358 for (i = 0; i < h; i++) {

359 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2,

360 w * c->bpp2);

361 outptr += c->pic->linesize[0];

362 }

363 }

364 }

365 bytestream2_skip(gb, 2);

366 chunks = bytestream2_get_be16(gb);

367 while (chunks--) {

368 if (bytestream2_get_bytes_left(gb) < 12) {

369 av_log(avctx, AV_LOG_ERROR, "Premature end of data!\n");

370 return -1;

371 }

372 dx = bytestream2_get_be16(gb);

373 dy = bytestream2_get_be16(gb);

374 w = bytestream2_get_be16(gb);

375 h = bytestream2_get_be16(gb);

376 enc = bytestream2_get_be32(gb);

377 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];

378 size_left = bytestream2_get_bytes_left(gb);

379 switch (enc) {

380 case MAGIC_WMVd: // cursor

381 if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) {

382 av_log(avctx, AV_LOG_ERROR, "dimensions too large\n");

383 return AVERROR_INVALIDDATA;

384 }

385 if (size_left < 2 + w * h * c->bpp2 * 2) {

386 av_log(avctx, AV_LOG_ERROR,

387 "Premature end of data! (need %i got %i)\n",

388 2 + w * h * c->bpp2 * 2, size_left);

389 return AVERROR_INVALIDDATA;

390 }

391 bytestream2_skip(gb, 2);

392 c->cur_w = w;

393 c->cur_h = h;

394 c->cur_hx = dx;

395 c->cur_hy = dy;

396 if ((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {

397 av_log(avctx, AV_LOG_ERROR,

398 "Cursor hot spot is not in image: "

399 "%ix%i of %ix%i cursor size\n",

400 c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);

401 c->cur_hx = c->cur_hy = 0;

402 }

403 if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) {

404 reset_buffers(c);

405 return AVERROR(EINVAL);

406 } else {

407 int screen_size = c->cur_w * c->cur_h * c->bpp2;

408 if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 ||

409 (ret = av_reallocp(&c->curmask, screen_size)) < 0 ||

410 (ret = av_reallocp(&c->screendta, screen_size)) < 0) {

411 reset_buffers(c);

412 return ret;

413 }

414 }

415 load_cursor(c);

416 break;

417 case MAGIC_WMVe: // unknown

418 bytestream2_skip(gb, 2);

419 break;

420 case MAGIC_WMVf: // update cursor position

421 c->cur_x = dx - c->cur_hx;

422 c->cur_y = dy - c->cur_hy;

423 break;

424 case MAGIC_WMVg: // unknown

425 bytestream2_skip(gb, 10);

426 break;

427 case MAGIC_WMVh: // unknown

428 bytestream2_skip(gb, 4);

429 break;

430 case MAGIC_WMVi: // ServerInitialization struct

431 c->pic->key_frame = 1;

432 c->pic->pict_type = AV_PICTURE_TYPE_I;

433 depth = bytestream2_get_byte(gb);

434 if (depth != c->bpp) {

435 av_log(avctx, AV_LOG_INFO,

436 "Depth mismatch. Container %i bpp, "

437 "Frame data: %i bpp\n",

438 c->bpp, depth);

439 }

440 bytestream2_skip(gb, 1);

441 c->bigendian = bytestream2_get_byte(gb);

442 if (c->bigendian & (~1)) {

443 av_log(avctx, AV_LOG_INFO,

444 "Invalid header: bigendian flag = %i\n", c->bigendian);

445 return AVERROR_INVALIDDATA;

446 }

447 //skip the rest of pixel format data

448 bytestream2_skip(gb, 13);

449 break;

450 case MAGIC_WMVj: // unknown

451 bytestream2_skip(gb, 2);

452 break;

453 case 0x00000000: // raw rectangle data

454 if ((dx + w > c->width) || (dy + h > c->height)) {

455 av_log(avctx, AV_LOG_ERROR,

456 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",

457 w, h, dx, dy, c->width, c->height);

458 return AVERROR_INVALIDDATA;

459 }

460 if (size_left < w * h * c->bpp2) {

461 av_log(avctx, AV_LOG_ERROR,

462 "Premature end of data! (need %i got %i)\n",

463 w * h * c->bpp2, size_left);

464 return AVERROR_INVALIDDATA;

465 }

466 paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian,

467 c->pic->linesize[0]);

468 break;

469 case 0x00000005: // HexTile encoded rectangle

470 if ((dx + w > c->width) || (dy + h > c->height)) {

471 av_log(avctx, AV_LOG_ERROR,

472 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",

473 w, h, dx, dy, c->width, c->height);

474 return AVERROR_INVALIDDATA;

475 }

476 res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]);

477 if (res < 0)

478 return res;

479 break;

480 default:

481 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);

482 chunks = 0; // leave chunks decoding loop

483 }

484 }

485 if (c->screendta) {

486 int i;

487 // save screen data before painting cursor

488 w = c->cur_w;

489 if (c->width < c->cur_x + w)

490 w = c->width - c->cur_x;

491 h = c->cur_h;

492 if (c->height < c->cur_y + h)

493 h = c->height - c->cur_y;

494 dx = c->cur_x;

495 if (dx < 0) {

496 w += dx;

497 dx = 0;

498 }

499 dy = c->cur_y;

500 if (dy < 0) {

501 h += dy;

502 dy = 0;

503 }

504 if ((w > 0) && (h > 0)) {

505 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];

506 for (i = 0; i < h; i++) {

507 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr,

508 w * c->bpp2);

509 outptr += c->pic->linesize[0];

510 }

511 outptr = c->pic->data[0];

512 put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y);

513 }

514 }

515 *got_frame = 1;

516 if ((ret = av_frame_ref(data, c->pic)) < 0)

517 return ret;

518

519 /* always report that the buffer was completely consumed */

520 return buf_size;

521 }

522

523 static av_cold int decode_init(AVCodecContext *avctx)

524 {

525 VmncContext * const c = avctx->priv_data;

526

527 c->avctx = avctx;

528 c->width = avctx->width;

529 c->height = avctx->height;

530 c->bpp = avctx->bits_per_coded_sample;

531 c->bpp2 = c->bpp / 8;

532

533 switch (c->bpp) {

534 case 8:

535 avctx->pix_fmt = AV_PIX_FMT_PAL8;

536 break;

537 case 16:

538 avctx->pix_fmt = AV_PIX_FMT_RGB555;

539 break;

540 case 32:

541 avctx->pix_fmt = AV_PIX_FMT_RGB32;

542 break;

543 default:

544 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);

545 return AVERROR_INVALIDDATA;

546 }

547

548 c->pic = av_frame_alloc();

549 if (!c->pic)

550 return AVERROR(ENOMEM);

551

552 return 0;

553 }

554

555 static av_cold int decode_end(AVCodecContext *avctx)

556 {

557 VmncContext * const c = avctx->priv_data;

558

559 av_frame_free(&c->pic);

560

561 av_freep(&c->curbits);

562 av_freep(&c->curmask);

563 av_freep(&c->screendta);

564 return 0;

565 }

566

567 AVCodec ff_vmnc_decoder = {

568 .name = "vmnc",

569 .long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),

570 .type = AVMEDIA_TYPE_VIDEO,

571 .id = AV_CODEC_ID_VMNC,

572 .priv_data_size = sizeof(VmncContext),

573 .init = decode_init,

574 .close = decode_end,

575 .decode = decode_frame,

576 .capabilities = CODEC_CAP_DR1,

577 };

Generated on Sun Mar 8 2015 02:35:02 for FFmpeg by doxygen 1.8.2