From 5f8690944a46c2435182c7a843b970966214962b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Xu=C3=A2n=20Minh?= Date: Thu, 28 Oct 2021 00:18:49 +0700 Subject: [PATCH 01/17] Add generic function fox SMX file --- .../convert/value_object/read/media/smx.pyx | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index d1f61a2735..10ad0c3364 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -59,6 +59,325 @@ cdef public dict LAYER_TYPES = { 1: SMXLayerType.SHADOW, 2: SMXLayerType.OUTLINE, } +cdef class SMXMainLayer8to5Variant: + pass + +cdef class SMXMainLayer4plus1Variant: + pass + +cdef class SMXOutlineLayerVariant: + pass + +cdef class SMXShadowLayerVariant: + pass + + +ctypedef fused SMXLayerVariant: + SMXMainLayer8to5Variant + SMXMainLayer4plus1Variant + SMXOutlineLayerVariant + SMXShadowLayerVariant + + +cdef process_drawing_cmds(SMXLayerVariant variant, + const uint8_t[:] &data_raw, + vector[pixel] &row_data, + Py_ssize_t rowid, + Py_ssize_t first_cmd_offset, + Py_ssize_t first_color_offset, + int chunk_pos, + size_t expected_size): + """ + TODO: docstring + """ + + # position in the command array, we start at the first command of this row + cdef Py_ssize_t dpos_cmd = first_cmd_offset + + # is the end of the current row reached? + cdef bool eor = False + + cdef uint8_t cmd = 0 + cdef uint8_t lower_crumb = 0 + cdef int pixel_count = 0 + + if SMXLayerVariant is SMXMainLayer8to5Variant + # Position in the pixel data array + cdef Py_ssize_t dpos_color = first_color_offset + + # Position in the compression chunk. + cdef bool odd = chunk_pos + cdef int px_dpos = 0 # For loop iterator + + cdef vector[uint8_t] pixel_data + pixel_data.reserve(4) + + # Pixel data temporary values that need further decompression + cdef uint8_t pixel_data_odd_0 = 0 + cdef uint8_t pixel_data_odd_1 = 0 + cdef uint8_t pixel_data_odd_2 = 0 + cdef uint8_t pixel_data_odd_3 = 0 + + # Mask for even indices + # cdef uint8_t pixel_mask_even_0 = 0xFF + cdef uint8_t pixel_mask_even_1 = 0b00000011 + cdef uint8_t pixel_mask_even_2 = 0b11110000 + cdef uint8_t pixel_mask_even_3 = 0b00111111 + + if SMXLayerVariant is SMXMainLayer4plus1Variant: + # Position in the pixel data array + cdef Py_ssize_t dpos_color = first_color_offset + + # Position in the compression chunk + cdef uint8_t dpos_chunk = chunk_pos + + cdef uint8_t palette_section_block = 0 + cdef uint8_t palette_section = 0 + + if SMXLayerVariant in (SMXShadowLayerVariant, SMXOutlineLayerVariant): + cdef uint8_t nextbyte = 0 + + # work through commands till end of row. + while not eor: + if row_data.size() > expected_size: + raise Exception( + f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + f"already!" + ) + + # fetch drawing instruction + cmd = data_raw[dpos_cmd] + + # Last 2 bits store command type + lower_crumb = 0b00000011 & cmd + + if lower_crumb == 0b00000011: + # eor (end of row) command, this row is finished now. + eor = True + dpos_cmd += 1 + + if is SMXShadowLayerVariant: + # shadows sometimes need an extra pixel at + # the end + if row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + continue + + elif lower_crumb == 0b00000000: + # skip command + # draw 'count' transparent pixels + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + elif lower_crumb == 0b00000001: + # color_list command + # draw the following 'count' pixels + # pixels are stored in 5 byte chunks + # even pixel indices have their info stored + # in byte[0] - byte[3]. odd pixel indices have + # their info stored in byte[1] - byte[4]. + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + if is SMXMainLayer8to5Variant: + for _ in range(pixel_count): + # Start fetching pixel data + if odd: + # Odd indices require manual extraction of each of the 4 values + + # Palette index. Essentially a rotation of (byte[1]byte[2]) + # by 6 to the left, then masking with 0x00FF. + pixel_data_odd_0 = data_raw[dpos_color + 1] + pixel_data_odd_1 = data_raw[dpos_color + 2] + pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) + + # Palette section. Described in byte[2] in bits 4-5. + pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) + + # Damage mask 1. Essentially a rotation of (byte[3]byte[4]) + # by 6 to the left, then masking with 0x00F0. + pixel_data_odd_2 = data_raw[dpos_color + 3] + pixel_data_odd_3 = data_raw[dpos_color + 4] + pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) + + # Damage mask 2. Described in byte[4] in bits 0-5. + pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) + + row_data.push_back(pixel(color_standard, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3])) + + # Go to next pixel + dpos_color += 5 + + else: + # Even indices can be read "as is". They just have to be masked. + for px_dpos in range(4): + pixel_data.push_back(data_raw[dpos_color + px_dpos]) + + row_data.push_back(pixel(color_standard, + pixel_data[0], + pixel_data[1] & pixel_mask_even_1, + pixel_data[2] & pixel_mask_even_2, + pixel_data[3] & pixel_mask_even_3)) + + odd = not odd + pixel_data.clear() + + if is SMXMainLayer4plus1Variant: + palette_section_block = data_raw[dpos_color + (4 - dpos_chunk)] + + for _ in range(pixel_count): + # Start fetching pixel data + palette_section = (palette_section_block >> (2 * dpos_chunk)) & 0x03 + row_data.push_back(pixel(color_standard, + data_raw[dpos_color], + palette_section, + 0, + 0)) + + dpos_color += 1 + dpos_chunk += 1 + + # Skip to next chunk + if dpos_chunk > 3: + dpos_chunk = 0 + dpos_color += 1 # Skip palette section block + palette_section_block = data_raw[dpos_color + 4] + + if SMXLayerVariant is SMXShadowLayerVariant: + for _ in range(pixel_count): + dpos_color += 1 + nextbyte = data_raw[dpos_color] + + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + + if SMXLayerVariant is SMXOutlineLayerVariant: + # we don't know the color the game wants + # so we just draw index 0 + row_data.push_back(pixel(color_outline, + 0, 0, 0, 0)) + + + elif lower_crumb == 0b00000010: + if SMXLayerVariant is SMXMainLayer8to5Variant: + # player_color command + # draw the following 'count' pixels + # pixels are stored in 5 byte chunks + # even pixel indices have their info stored + # in byte[0] - byte[3]. odd pixel indices have + # their info stored in byte[1] - byte[4]. + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + # Start fetching pixel data + if odd: + # Odd indices require manual extraction of each of the 4 values + + # Palette index. Essentially a rotation of (byte[1]byte[2]) + # by 6 to the left, then masking with 0x00FF. + pixel_data_odd_0 = data_raw[dpos_color + 1] + pixel_data_odd_1 = data_raw[dpos_color + 2] + pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) + + # Palette section. Described in byte[2] in bits 4-5. + pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) + + # Damage modifier 1. Essentially a rotation of (byte[3]byte[4]) + # by 6 to the left, then masking with 0x00F0. + pixel_data_odd_2 = data_raw[dpos_color + 3] + pixel_data_odd_3 = data_raw[dpos_color + 4] + pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) + + # Damage modifier 2. Described in byte[4] in bits 0-5. + pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3])) + + # Go to next pixel + dpos_color += 5 + + else: + # Even indices can be read "as is". They just have to be masked. + for px_dpos in range(4): + pixel_data.push_back(data_raw[dpos_color + px_dpos]) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1] & pixel_mask_even_1, + pixel_data[2] & pixel_mask_even_2, + pixel_data[3] & pixel_mask_even_3)) + + odd = not odd + pixel_data.clear() + + + elif lower_crumb == 0b00000010: + if SMXLayerVariant is SMXMainLayer4plus1Variant: + # player_color command + # draw the following 'count' pixels + # 4 pixels are stored in every 5 byte chunk. + # palette indices are contained in byte[0] - byte[3] + # palette sections are stored in byte[4] + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + # Start fetching pixel data + palette_section = (palette_section_block >> (2 * dpos_chunk)) & 0x03 + row_data.push_back(pixel(color_player, + data_raw[dpos_color], + palette_section, + 0, + 0)) + + dpos_color += 1 + dpos_chunk += 1 + + # Skip to next chunk + if dpos_chunk > 3: + dpos_chunk = 0 + dpos_color += 1 # Skip palette section block + palette_section_block = data_raw[dpos_color + 4] + + if SMXLayerVariant is (SMXOutlineLayerVariant, SMXShadowLayerVariant): + pass + + else: + raise Exception( + f"unknown smx main graphics layer drawing command: " + + f"{cmd:#x} in row {rowid:d}" + ) + + # Process next command + dpos_cmd += 1 + + if SMXLayerVariant in (SMXMainLayer8to5Variant, SMXMainLayer4plus1Variant): + return dpos_cmd, dpos_color, odd, row_data + if SMXLayerVariant in (SMXOutlineLayerVariant, SMXShadowLayerVariant): + return dpos_cmd, dpos_cmd, chunk_pos, row_data class SMX: From b421336fe8a70460cf7ce45955beed6a2e8449e0 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 27 Dec 2024 15:23:22 +0000 Subject: [PATCH 02/17] compiling code --- .../convert/value_object/read/media/smx.pyx | 688 ++---------------- 1 file changed, 55 insertions(+), 633 deletions(-) diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index 10ad0c3364..b4cee859dc 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -59,28 +59,29 @@ cdef public dict LAYER_TYPES = { 1: SMXLayerType.SHADOW, 2: SMXLayerType.OUTLINE, } -cdef class SMXMainLayer8to5Variant: +cdef class SMXMainLayer8to5: pass -cdef class SMXMainLayer4plus1Variant: +cdef class SMXMainLayer4plus1: pass -cdef class SMXOutlineLayerVariant: +cdef class SMXOutlineLayer: pass -cdef class SMXShadowLayerVariant: +cdef class SMXShadowLayer: pass ctypedef fused SMXLayerVariant: - SMXMainLayer8to5Variant - SMXMainLayer4plus1Variant - SMXOutlineLayerVariant - SMXShadowLayerVariant + SMXLayer + SMXMainLayer8to5 + SMXMainLayer4plus1 + SMXOutlineLayer + SMXShadowLayer - -cdef process_drawing_cmds(SMXLayerVariant variant, - const uint8_t[:] &data_raw, +@cython.boundscheck(False) +cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant variant, + const uint8_t[::1] &data_raw, vector[pixel] &row_data, Py_ssize_t rowid, Py_ssize_t first_cmd_offset, @@ -100,49 +101,34 @@ cdef process_drawing_cmds(SMXLayerVariant variant, cdef uint8_t cmd = 0 cdef uint8_t lower_crumb = 0 cdef int pixel_count = 0 - - if SMXLayerVariant is SMXMainLayer8to5Variant + cdef Py_ssize_t dpos_color = 0 + cdef vector[uint8_t] pixel_data + # Pixel data temporary values that need further decompression + cdef uint8_t pixel_data_odd_0 = 0 + cdef uint8_t pixel_data_odd_1 = 0 + cdef uint8_t pixel_data_odd_2 = 0 + cdef uint8_t pixel_data_odd_3 = 0 + # Mask for even indices + # cdef uint8_t pixel_mask_even_0 = 0xFF + cdef uint8_t pixel_mask_even_1 = 0b00000011 + cdef uint8_t pixel_mask_even_2 = 0b11110000 + cdef uint8_t pixel_mask_even_3 = 0b00111111 + + if variant in (SMXMainLayer8to5, SMXMainLayer4plus1): # Position in the pixel data array - cdef Py_ssize_t dpos_color = first_color_offset + dpos_color = first_color_offset - # Position in the compression chunk. - cdef bool odd = chunk_pos - cdef int px_dpos = 0 # For loop iterator - - cdef vector[uint8_t] pixel_data - pixel_data.reserve(4) - - # Pixel data temporary values that need further decompression - cdef uint8_t pixel_data_odd_0 = 0 - cdef uint8_t pixel_data_odd_1 = 0 - cdef uint8_t pixel_data_odd_2 = 0 - cdef uint8_t pixel_data_odd_3 = 0 - - # Mask for even indices - # cdef uint8_t pixel_mask_even_0 = 0xFF - cdef uint8_t pixel_mask_even_1 = 0b00000011 - cdef uint8_t pixel_mask_even_2 = 0b11110000 - cdef uint8_t pixel_mask_even_3 = 0b00111111 - - if SMXLayerVariant is SMXMainLayer4plus1Variant: - # Position in the pixel data array - cdef Py_ssize_t dpos_color = first_color_offset - # Position in the compression chunk - cdef uint8_t dpos_chunk = chunk_pos - - cdef uint8_t palette_section_block = 0 - cdef uint8_t palette_section = 0 - - if SMXLayerVariant in (SMXShadowLayerVariant, SMXOutlineLayerVariant): - cdef uint8_t nextbyte = 0 + cdef uint8_t palette_section_block = 0 + cdef uint8_t palette_section = 0 + cdef uint8_t nextbyte = 0 # work through commands till end of row. while not eor: if row_data.size() > expected_size: raise Exception( f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + f"with layer type {variant:#x}, but we have {row_data.size():d} " f"already!" ) @@ -157,7 +143,7 @@ cdef process_drawing_cmds(SMXLayerVariant variant, eor = True dpos_cmd += 1 - if is SMXShadowLayerVariant: + if variant is SMXShadowLayer: # shadows sometimes need an extra pixel at # the end if row_data.size() < expected_size: @@ -191,10 +177,11 @@ cdef process_drawing_cmds(SMXLayerVariant variant, pixel_count = (cmd >> 2) + 1 - if is SMXMainLayer8to5Variant: + if variant is SMXMainLayer8to5: + pixel_data.reserve(4) for _ in range(pixel_count): # Start fetching pixel data - if odd: + if chunk_pos: # Odd indices require manual extraction of each of the 4 values # Palette index. Essentially a rotation of (byte[1]byte[2]) @@ -226,8 +213,8 @@ cdef process_drawing_cmds(SMXLayerVariant variant, else: # Even indices can be read "as is". They just have to be masked. - for px_dpos in range(4): - pixel_data.push_back(data_raw[dpos_color + px_dpos]) + for i in range(4): + pixel_data.push_back(data_raw[dpos_color + i]) row_data.push_back(pixel(color_standard, pixel_data[0], @@ -238,12 +225,12 @@ cdef process_drawing_cmds(SMXLayerVariant variant, odd = not odd pixel_data.clear() - if is SMXMainLayer4plus1Variant: - palette_section_block = data_raw[dpos_color + (4 - dpos_chunk)] + if variant is SMXMainLayer4plus1: + palette_section_block = data_raw[dpos_color + (4 - chunk_pos)] for _ in range(pixel_count): # Start fetching pixel data - palette_section = (palette_section_block >> (2 * dpos_chunk)) & 0x03 + palette_section = (palette_section_block >> (2 * chunk_pos)) & 0x03 row_data.push_back(pixel(color_standard, data_raw[dpos_color], palette_section, @@ -251,15 +238,15 @@ cdef process_drawing_cmds(SMXLayerVariant variant, 0)) dpos_color += 1 - dpos_chunk += 1 + chunk_pos += 1 # Skip to next chunk - if dpos_chunk > 3: - dpos_chunk = 0 + if chunk_pos > 3: + chunk_pos = 0 dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] - if SMXLayerVariant is SMXShadowLayerVariant: + if variant is SMXShadowLayer: for _ in range(pixel_count): dpos_color += 1 nextbyte = data_raw[dpos_color] @@ -267,7 +254,7 @@ cdef process_drawing_cmds(SMXLayerVariant variant, row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) - if SMXLayerVariant is SMXOutlineLayerVariant: + if variant is SMXOutlineLayer: # we don't know the color the game wants # so we just draw index 0 row_data.push_back(pixel(color_outline, @@ -275,7 +262,7 @@ cdef process_drawing_cmds(SMXLayerVariant variant, elif lower_crumb == 0b00000010: - if SMXLayerVariant is SMXMainLayer8to5Variant: + if variant is SMXMainLayer8to5: # player_color command # draw the following 'count' pixels # pixels are stored in 5 byte chunks @@ -334,7 +321,7 @@ cdef process_drawing_cmds(SMXLayerVariant variant, elif lower_crumb == 0b00000010: - if SMXLayerVariant is SMXMainLayer4plus1Variant: + if variant is SMXMainLayer4plus1: # player_color command # draw the following 'count' pixels # 4 pixels are stored in every 5 byte chunk. @@ -346,7 +333,7 @@ cdef process_drawing_cmds(SMXLayerVariant variant, for _ in range(pixel_count): # Start fetching pixel data - palette_section = (palette_section_block >> (2 * dpos_chunk)) & 0x03 + palette_section = (palette_section_block >> (2 * chunk_pos)) & 0x03 row_data.push_back(pixel(color_player, data_raw[dpos_color], palette_section, @@ -354,15 +341,15 @@ cdef process_drawing_cmds(SMXLayerVariant variant, 0)) dpos_color += 1 - dpos_chunk += 1 + chunk_pos += 1 # Skip to next chunk - if dpos_chunk > 3: - dpos_chunk = 0 + if chunk_pos > 3: + chunk_pos = 0 dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] - if SMXLayerVariant is (SMXOutlineLayerVariant, SMXShadowLayerVariant): + if variant is (SMXOutlineLayer, SMXShadowLayer): pass else: @@ -374,9 +361,9 @@ cdef process_drawing_cmds(SMXLayerVariant variant, # Process next command dpos_cmd += 1 - if SMXLayerVariant in (SMXMainLayer8to5Variant, SMXMainLayer4plus1Variant): + if variant in (SMXMainLayer8to5, SMXMainLayer4plus1): return dpos_cmd, dpos_color, odd, row_data - if SMXLayerVariant in (SMXOutlineLayerVariant, SMXShadowLayerVariant): + if variant in (SMXOutlineLayer, SMXShadowLayer): return dpos_cmd, dpos_cmd, chunk_pos, row_data @@ -729,7 +716,7 @@ cdef class SMXLayer: # process the drawing commands for this row. next_cmd_offset, next_color_offset, chunk_pos, row_data = \ - self.process_drawing_cmds( + process_drawing_cmds(self, data_raw, row_data, rowid, @@ -757,26 +744,6 @@ cdef class SMXLayer: return next_cmd_offset, next_color_offset, chunk_pos, row_data - cdef (int, int, int, vector[pixel]) process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): - """ - Extracts pixel data from the layer data. Every layer type uses - its own implementation for better optimization. - - :param row_data: Pixel data is appended to this array. - :param rowid: Index of the current row in the layer. - :param first_cmd_offset: Offset of the first command of the current row. - :param first_color_offset: Offset of the first pixel data value of the current row. - :param chunk_pos: Current position in the compressed chunk. - :param expected_size: Expected length of row_data after encountering the EOR command. - """ - pass def get_picture_data(self, palette): """ @@ -811,551 +778,6 @@ cdef class SMXLayer: return repr(self.info) -cdef class SMXMainLayer8to5(SMXLayer): - """ - Compressed SMP layer (compression type 8to5) for the main graphics sprite. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands that were - compressed with 8to5 compression. - """ - # position in the command array, we start at the first command of this row - cdef Py_ssize_t dpos_cmd = first_cmd_offset - - # Position in the pixel data array - cdef Py_ssize_t dpos_color = first_color_offset - - # Position in the compression chunk. - cdef bool odd = chunk_pos - cdef int px_dpos = 0 # For loop iterator - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd = 0 - cdef uint8_t lower_crumb = 0 - cdef int pixel_count = 0 - cdef vector[uint8_t] pixel_data - pixel_data.reserve(4) - - # Pixel data temporary values that need further decompression - cdef uint8_t pixel_data_odd_0 = 0 - cdef uint8_t pixel_data_odd_1 = 0 - cdef uint8_t pixel_data_odd_2 = 0 - cdef uint8_t pixel_data_odd_3 = 0 - - # Mask for even indices - # cdef uint8_t pixel_mask_even_0 = 0xFF - cdef uint8_t pixel_mask_even_1 = 0b00000011 - cdef uint8_t pixel_mask_even_2 = 0b11110000 - cdef uint8_t pixel_mask_even_3 = 0b00111111 - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos_cmd] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - if lower_crumb == 0b00000011: - # eor (end of row) command, this row is finished now. - eor = True - dpos_cmd += 1 - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # pixels are stored in 5 byte chunks - # even pixel indices have their info stored - # in byte[0] - byte[3]. odd pixel indices have - # their info stored in byte[1] - byte[4]. - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # Start fetching pixel data - if odd: - # Odd indices require manual extraction of each of the 4 values - - # Palette index. Essentially a rotation of (byte[1]byte[2]) - # by 6 to the left, then masking with 0x00FF. - pixel_data_odd_0 = data_raw[dpos_color + 1] - pixel_data_odd_1 = data_raw[dpos_color + 2] - pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) - - # Palette section. Described in byte[2] in bits 4-5. - pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) - - # Damage mask 1. Essentially a rotation of (byte[3]byte[4]) - # by 6 to the left, then masking with 0x00F0. - pixel_data_odd_2 = data_raw[dpos_color + 3] - pixel_data_odd_3 = data_raw[dpos_color + 4] - pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) - - # Damage mask 2. Described in byte[4] in bits 0-5. - pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) - - row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3])) - - # Go to next pixel - dpos_color += 5 - - else: - # Even indices can be read "as is". They just have to be masked. - for px_dpos in range(4): - pixel_data.push_back(data_raw[dpos_color + px_dpos]) - - row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1] & pixel_mask_even_1, - pixel_data[2] & pixel_mask_even_2, - pixel_data[3] & pixel_mask_even_3)) - - odd = not odd - pixel_data.clear() - - elif lower_crumb == 0b00000010: - # player_color command - # draw the following 'count' pixels - # pixels are stored in 5 byte chunks - # even pixel indices have their info stored - # in byte[0] - byte[3]. odd pixel indices have - # their info stored in byte[1] - byte[4]. - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # Start fetching pixel data - if odd: - # Odd indices require manual extraction of each of the 4 values - - # Palette index. Essentially a rotation of (byte[1]byte[2]) - # by 6 to the left, then masking with 0x00FF. - pixel_data_odd_0 = data_raw[dpos_color + 1] - pixel_data_odd_1 = data_raw[dpos_color + 2] - pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) - - # Palette section. Described in byte[2] in bits 4-5. - pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) - - # Damage modifier 1. Essentially a rotation of (byte[3]byte[4]) - # by 6 to the left, then masking with 0x00F0. - pixel_data_odd_2 = data_raw[dpos_color + 3] - pixel_data_odd_3 = data_raw[dpos_color + 4] - pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) - - # Damage modifier 2. Described in byte[4] in bits 0-5. - pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) - - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3])) - - # Go to next pixel - dpos_color += 5 - - else: - # Even indices can be read "as is". They just have to be masked. - for px_dpos in range(4): - pixel_data.push_back(data_raw[dpos_color + px_dpos]) - - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1] & pixel_mask_even_1, - pixel_data[2] & pixel_mask_even_2, - pixel_data[3] & pixel_mask_even_3)) - - odd = not odd - pixel_data.clear() - - else: - raise Exception( - f"unknown smx main graphics layer drawing command: " + - f"{cmd:#x} in row {rowid:d}" - ) - - # Process next command - dpos_cmd += 1 - - return dpos_cmd, dpos_color, odd, row_data - - -cdef class SMXMainLayer4plus1(SMXLayer): - """ - Compressed SMP layer (compression type 4plus1) for the main graphics sprite. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands that were - compressed with 4plus1 compression. - """ - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos_cmd = first_cmd_offset - - # Position in the pixel data array - cdef Py_ssize_t dpos_color = first_color_offset - - # Position in the compression chunk - cdef uint8_t dpos_chunk = chunk_pos - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd = 0 - cdef uint8_t lower_crumb = 0 - cdef int pixel_count = 0 - cdef uint8_t palette_section_block = 0 - cdef uint8_t palette_section = 0 - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos_cmd] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - if lower_crumb == 0b00000011: - # eor (end of row) command, this row is finished now. - eor = True - dpos_cmd += 1 - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # 4 pixels are stored in every 5 byte chunk. - # palette indices are contained in byte[0] - byte[3] - # palette sections are stored in byte[4] - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - palette_section_block = data_raw[dpos_color + (4 - dpos_chunk)] - - for _ in range(pixel_count): - # Start fetching pixel data - palette_section = (palette_section_block >> (2 * dpos_chunk)) & 0x03 - row_data.push_back(pixel(color_standard, - data_raw[dpos_color], - palette_section, - 0, - 0)) - - dpos_color += 1 - dpos_chunk += 1 - - # Skip to next chunk - if dpos_chunk > 3: - dpos_chunk = 0 - dpos_color += 1 # Skip palette section block - palette_section_block = data_raw[dpos_color + 4] - - elif lower_crumb == 0b00000010: - # player_color command - # draw the following 'count' pixels - # 4 pixels are stored in every 5 byte chunk. - # palette indices are contained in byte[0] - byte[3] - # palette sections are stored in byte[4] - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # Start fetching pixel data - palette_section = (palette_section_block >> (2 * dpos_chunk)) & 0x03 - row_data.push_back(pixel(color_player, - data_raw[dpos_color], - palette_section, - 0, - 0)) - - dpos_color += 1 - dpos_chunk += 1 - - # Skip to next chunk - if dpos_chunk > 3: - dpos_chunk = 0 - dpos_color += 1 # Skip palette section block - palette_section_block = data_raw[dpos_color + 4] - - else: - raise Exception( - f"unknown smx main graphics layer drawing command: " + - f"{cmd:#x} in row {rowid:d}" - ) - - # Process next command - dpos_cmd += 1 - - return dpos_cmd, dpos_color, dpos_chunk, row_data - - -cdef class SMXShadowLayer(SMXLayer): - """ - Compressed SMP layer for the shadow graphics. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands - found for this row in the SMX layer. - """ - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd = 0 - cdef uint8_t nextbyte = 0 - cdef uint8_t lower_crumb = 0 - cdef int pixel_count = 0 - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn ifn row {rowid:d} " + - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - if lower_crumb == 0b00000011: - # eol (end of line) command, this row is finished now. - eor = True - dpos += 1 - - # shadows sometimes need an extra pixel at - # the end - if row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # pixels are stored as 1 byte alpha values - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - dpos += 1 - nextbyte = data_raw[dpos] - - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - - else: - raise Exception( - f"unknown smp shadow layer drawing command: " + - f"{cmd:#x} in row {rowid}" - ) - - # process next command - dpos += 1 - - # end of row reached, return the created pixel array. - return dpos, dpos, chunk_pos, row_data - - -cdef class SMXOutlineLayer(SMXLayer): - """ - Compressed SMP layer for the outline graphics. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands - found for this row in the SMX layer. - """ - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd = 0 - cdef uint8_t nextbyte = 0 - cdef uint8_t lower_crumb = 0 - cdef int pixel_count = 0 - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - # opcode: cmd, rowid: rowid - - if lower_crumb == 0b00000011: - # eol (end of line) command, this row is finished now. - eor = True - dpos += 1 - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # as player outline colors. - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # we don't know the color the game wants - # so we just draw index 0 - row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) - - else: - raise Exception( - f"unknown smp outline layer drawing command: " + - f"{cmd:#x} in row {rowid}" - ) - - # process next command - dpos += 1 - - # end of row reached, return the created pixel array. - return dpos, dpos, chunk_pos, row_data @cython.boundscheck(False) From 7177fd2e9a72b0364c401fdc33fd4a87e6784e6e Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 27 Dec 2024 19:30:30 +0000 Subject: [PATCH 03/17] format --- .../convert/value_object/read/media/smx.pyx | 181 +++++++++--------- 1 file changed, 95 insertions(+), 86 deletions(-) diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index b4cee859dc..bcf9ec0e33 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -2,6 +2,9 @@ # # cython: infer_types=True +from libcpp.vector cimport vector +from libcpp cimport bool +from libc.stdint cimport uint8_t, uint16_t from enum import Enum import numpy from struct import Struct, unpack_from @@ -12,11 +15,6 @@ from .....log import spam, dbg cimport cython cimport numpy -from libc.stdint cimport uint8_t, uint16_t -from libcpp cimport bool -from libcpp.vector cimport vector - - # SMX files have little endian byte order endianness = "< " @@ -49,8 +47,8 @@ class SMXLayerType(Enum): """ SMX layer types. """ - MAIN = "main" - SHADOW = "shadow" + MAIN = "main" + SHADOW = "shadow" OUTLINE = "outline" @@ -79,15 +77,16 @@ ctypedef fused SMXLayerVariant: SMXOutlineLayer SMXShadowLayer + @cython.boundscheck(False) -cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant variant, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): +cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant variant, + const uint8_t[::1] & data_raw, + vector[pixel] & row_data, + Py_ssize_t rowid, + Py_ssize_t first_cmd_offset, + Py_ssize_t first_color_offset, + int chunk_pos, + size_t expected_size): """ TODO: docstring """ @@ -118,11 +117,10 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # Position in the pixel data array dpos_color = first_color_offset - cdef uint8_t palette_section_block = 0 cdef uint8_t palette_section = 0 cdef uint8_t nextbyte = 0 - + # work through commands till end of row. while not eor: if row_data.size() > expected_size: @@ -176,7 +174,7 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # count = (cmd >> 2) + 1 pixel_count = (cmd >> 2) + 1 - + if variant is SMXMainLayer8to5: pixel_data.reserve(4) for _ in range(pixel_count): @@ -188,7 +186,8 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # by 6 to the left, then masking with 0x00FF. pixel_data_odd_0 = data_raw[dpos_color + 1] pixel_data_odd_1 = data_raw[dpos_color + 2] - pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) + pixel_data.push_back( + (pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) # Palette section. Described in byte[2] in bits 4-5. pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) @@ -197,16 +196,17 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # by 6 to the left, then masking with 0x00F0. pixel_data_odd_2 = data_raw[dpos_color + 3] pixel_data_odd_3 = data_raw[dpos_color + 4] - pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) + pixel_data.push_back( + ((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) # Damage mask 2. Described in byte[4] in bits 0-5. pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3])) + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3])) # Go to next pixel dpos_color += 5 @@ -217,10 +217,10 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant pixel_data.push_back(data_raw[dpos_color + i]) row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1] & pixel_mask_even_1, - pixel_data[2] & pixel_mask_even_2, - pixel_data[3] & pixel_mask_even_3)) + pixel_data[0], + pixel_data[1] & pixel_mask_even_1, + pixel_data[2] & pixel_mask_even_2, + pixel_data[3] & pixel_mask_even_3)) odd = not odd pixel_data.clear() @@ -230,7 +230,8 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant for _ in range(pixel_count): # Start fetching pixel data - palette_section = (palette_section_block >> (2 * chunk_pos)) & 0x03 + palette_section = ( + palette_section_block >> (2 * chunk_pos)) & 0x03 row_data.push_back(pixel(color_standard, data_raw[dpos_color], palette_section, @@ -243,7 +244,7 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # Skip to next chunk if chunk_pos > 3: chunk_pos = 0 - dpos_color += 1 # Skip palette section block + dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] if variant is SMXShadowLayer: @@ -258,9 +259,8 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # we don't know the color the game wants # so we just draw index 0 row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) + 0, 0, 0, 0)) - elif lower_crumb == 0b00000010: if variant is SMXMainLayer8to5: # player_color command @@ -282,7 +282,8 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # by 6 to the left, then masking with 0x00FF. pixel_data_odd_0 = data_raw[dpos_color + 1] pixel_data_odd_1 = data_raw[dpos_color + 2] - pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) + pixel_data.push_back( + (pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) # Palette section. Described in byte[2] in bits 4-5. pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) @@ -291,16 +292,17 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # by 6 to the left, then masking with 0x00F0. pixel_data_odd_2 = data_raw[dpos_color + 3] pixel_data_odd_3 = data_raw[dpos_color + 4] - pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) + pixel_data.push_back( + ((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) # Damage modifier 2. Described in byte[4] in bits 0-5. pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3])) + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3])) # Go to next pixel dpos_color += 5 @@ -308,18 +310,18 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant else: # Even indices can be read "as is". They just have to be masked. for px_dpos in range(4): - pixel_data.push_back(data_raw[dpos_color + px_dpos]) + pixel_data.push_back( + data_raw[dpos_color + px_dpos]) row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1] & pixel_mask_even_1, - pixel_data[2] & pixel_mask_even_2, - pixel_data[3] & pixel_mask_even_3)) + pixel_data[0], + pixel_data[1] & pixel_mask_even_1, + pixel_data[2] & pixel_mask_even_2, + pixel_data[3] & pixel_mask_even_3)) odd = not odd pixel_data.clear() - elif lower_crumb == 0b00000010: if variant is SMXMainLayer4plus1: # player_color command @@ -333,12 +335,13 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant for _ in range(pixel_count): # Start fetching pixel data - palette_section = (palette_section_block >> (2 * chunk_pos)) & 0x03 + palette_section = ( + palette_section_block >> (2 * chunk_pos)) & 0x03 row_data.push_back(pixel(color_player, - data_raw[dpos_color], - palette_section, - 0, - 0)) + data_raw[dpos_color], + palette_section, + 0, + 0)) dpos_color += 1 chunk_pos += 1 @@ -346,7 +349,7 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant # Skip to next chunk if chunk_pos > 3: chunk_pos = 0 - dpos_color += 1 # Skip palette section block + dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] if variant is (SMXOutlineLayer, SMXShadowLayer): @@ -356,7 +359,7 @@ cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant raise Exception( f"unknown smx main graphics layer drawing command: " + f"{cmd:#x} in row {rowid:d}" - ) + ) # Process next command dpos_cmd += 1 @@ -409,14 +412,16 @@ class SMX: """ smx_header = SMX.smx_header.unpack_from(data) - self.smp_type, version, frame_count, file_size_comp,\ + self.smp_type, version, frame_count, file_size_comp, \ file_size_uncomp, comment = smx_header dbg("SMX") dbg(" version: %s", version) dbg(" frame count: %s", frame_count) - dbg(" file size compressed: %s B", file_size_comp + 0x20) # 0x20 = SMX header size - dbg(" file size uncompressed: %s B", file_size_uncomp + 0x40) # 0x80 = SMP header size + # 0x20 = SMX header size + dbg(" file size compressed: %s B", file_size_comp + 0x20) + # 0x80 = SMP header size + dbg(" file size uncompressed: %s B", file_size_uncomp + 0x40) dbg(" comment: %s", comment.decode('ascii')) # SMX graphic frames are created from overlaying @@ -434,7 +439,7 @@ class SMX: frame_header = SMX.smx_frame_header.unpack_from( data, current_offset) - frame_type , palette_number, _ = frame_header + frame_type, palette_number, _ = frame_header current_offset += SMX.smx_frame_header.size @@ -453,7 +458,7 @@ class SMX: layer_header_data = SMX.smx_layer_header.unpack_from( data, current_offset) - width, height, hotspot_x, hotspot_y,\ + width, height, hotspot_x, hotspot_y, \ distance_next_frame, _ = layer_header_data current_offset += SMX.smx_layer_header.size @@ -463,12 +468,14 @@ class SMX: # Skip outline table current_offset += 4 * height - qdl_command_array_size = Struct("< I").unpack_from(data, current_offset)[0] + qdl_command_array_size = Struct( + "< I").unpack_from(data, current_offset)[0] current_offset += 4 # Read length of color table if layer_type is SMXLayerType.MAIN: - qdl_color_table_size = Struct("< I").unpack_from(data, current_offset)[0] + qdl_color_table_size = Struct( + "< I").unpack_from(data, current_offset)[0] current_offset += 4 qdl_color_table_offset = current_offset + qdl_command_array_size @@ -488,16 +495,20 @@ class SMX: if layer_type is SMXLayerType.MAIN: if layer_header.compression_type == 0x08: - self.main_frames.append(SMXMainLayer8to5(layer_header, data)) + self.main_frames.append( + SMXMainLayer8to5(layer_header, data)) elif layer_header.compression_type == 0x00: - self.main_frames.append(SMXMainLayer4plus1(layer_header, data)) + self.main_frames.append( + SMXMainLayer4plus1(layer_header, data)) elif layer_type is SMXLayerType.SHADOW: - self.shadow_frames.append(SMXShadowLayer(layer_header, data)) + self.shadow_frames.append( + SMXShadowLayer(layer_header, data)) elif layer_type is SMXLayerType.OUTLINE: - self.outline_frames.append(SMXOutlineLayer(layer_header, data)) + self.outline_frames.append( + SMXOutlineLayer(layer_header, data)) def get_frames(self, layer: int = 0): """ @@ -673,16 +684,17 @@ cdef class SMXLayer: # process cmd table for i in range(row_count): cmd_offset, color_offset, chunk_pos, row_data = \ - self.create_color_row(data_raw, i, cmd_offset, color_offset, chunk_pos) + self.create_color_row( + data_raw, i, cmd_offset, color_offset, chunk_pos) self.pcolor.push_back(row_data) - cdef inline (int, int, int, vector[pixel]) create_color_row(self, - const uint8_t[::1] &data_raw, - Py_ssize_t rowid, - int cmd_offset, - int color_offset, - int chunk_pos): + cdef inline(int, int, int, vector[pixel]) create_color_row(self, + const uint8_t[::1] & data_raw, + Py_ssize_t rowid, + int cmd_offset, + int color_offset, + int chunk_pos): """ Extract colors (pixels) for the given rowid. @@ -717,14 +729,14 @@ cdef class SMXLayer: # process the drawing commands for this row. next_cmd_offset, next_color_offset, chunk_pos, row_data = \ process_drawing_cmds(self, - data_raw, - row_data, - rowid, - first_cmd_offset, - first_color_offset, - chunk_pos, - pixel_count - bounds.right - ) + data_raw, + row_data, + rowid, + first_cmd_offset, + first_color_offset, + chunk_pos, + pixel_count - bounds.right + ) # finish by filling up the right transparent space for i in range(bounds.right): @@ -736,7 +748,7 @@ cdef class SMXLayer: summary = "%d/%d -> row %d, layer type %d, offset %d / %#x" % ( got, pixel_count, rowid, self.info.layer_type, first_cmd_offset, first_cmd_offset - ) + ) txt = "got %%s pixels than expected: %s, missing: %d" % ( summary, abs(pixel_count - got)) @@ -744,7 +756,6 @@ cdef class SMXLayer: return next_cmd_offset, next_color_offset, chunk_pos, row_data - def get_picture_data(self, palette): """ Convert the palette index matrix to a RGBA image. @@ -778,12 +789,10 @@ cdef class SMXLayer: return repr(self.info) - - @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, - numpy.ndarray[numpy.uint8_t, ndim=2, mode="c"] palette): +cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, + numpy.ndarray[numpy.uint8_t, ndim = 2, mode = "c"] palette): """ converts a palette index image matrix to an rgba matrix. @@ -794,7 +803,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t[:, ::1] m_lookup = palette @@ -883,7 +892,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): +cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix): """ converts the damage modifier values to an image using the RG values. @@ -893,7 +902,7 @@ cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t r = 0 From 74cb7d0cd7802c0b3fabe6c33ff7a221253d701b Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 27 Dec 2024 20:42:22 +0000 Subject: [PATCH 04/17] class defs --- .../convert/value_object/read/media/smx.pyx | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index bcf9ec0e33..2cb4749091 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -57,17 +57,21 @@ cdef public dict LAYER_TYPES = { 1: SMXLayerType.SHADOW, 2: SMXLayerType.OUTLINE, } -cdef class SMXMainLayer8to5: - pass +cdef class SMXMainLayer8to5(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) -cdef class SMXMainLayer4plus1: - pass +cdef class SMXMainLayer4plus1(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) -cdef class SMXOutlineLayer: - pass +cdef class SMXOutlineLayer(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) -cdef class SMXShadowLayer: - pass +cdef class SMXShadowLayer(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) ctypedef fused SMXLayerVariant: From b5427b57667a46223372946e5dc5f1a17f2ff0af Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 27 Dec 2024 22:42:20 +0000 Subject: [PATCH 05/17] smp fused --- .../convert/value_object/read/media/smp.pyx | 499 ++++++------------ 1 file changed, 169 insertions(+), 330 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 49a2122175..9454bd9c80 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -253,6 +253,170 @@ class SMPLayerHeader: ) return "".join(ret) + +cdef class SMPMainLayer(SMPLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + + def get_damage_mask(self): + """ + Convert the 4th pixel byte to a mask used for damaged units. + """ + return determine_damage_matrix(self.pcolor) + +cdef class SMPShadowLayer(SMPLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + +cdef class SMPOutlineLayer(SMPLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + + +ctypedef fused SMPLayerVariant: + SMPLayer + SMPMainLayer + SMPShadowLayer + SMPOutlineLayer + + +@cython.boundscheck(False) +cdef void process_drawing_cmds(SMPLayerVariant variant, + const uint8_t[::1] &data_raw, + vector[pixel] &row_data, + Py_ssize_t rowid, + Py_ssize_t first_cmd_offset, + size_t expected_size): + + """ + extract colors (pixels) for the drawing commands + found for this row in the SMP frame. + """ + + # position in the data blob, we start at the first command of this row + cdef Py_ssize_t dpos = first_cmd_offset + + # is the end of the current row reached? + cdef bool eor = False + + cdef uint8_t cmd = 0 + cdef uint8_t nextbyte = 0 + cdef uint8_t lower_crumb = 0 + cdef int pixel_count = 0 + + cdef vector[uint8_t] pixel_data + pixel_data.reserve(4) + + # work through commands till end of row. + while not eor: + if row_data.size() > expected_size: + raise Exception( + f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + + f"with layer type {variant.info.layer_type:#x}, but we have {row_data.size():d}" + ) + + # fetch drawing instruction + cmd = data_raw[dpos] + + # Last 2 bits store command type + lower_crumb = 0b00000011 & cmd + + # opcode: cmd, rowid: rowid + + if lower_crumb == 0b00000011: + # eol (end of line) command, this row is finished now. + eor = True + + if variant is SMPShadowLayer and row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) + + continue + + elif lower_crumb == 0b00000000: + # skip command + # draw 'count' transparent pixels + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + elif lower_crumb == 0b00000001: + # color_list command + # draw the following 'count' pixels + # pixels are stored as 4 byte palette and meta infos + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + if variant is SMPShadowLayer: + dpos += 1 + nextbyte = data_raw[dpos] + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + elif variant is SMPOutlineLayer: + # we don't know the color the game wants + # so we just draw index 0 + row_data.push_back(pixel(color_outline, + 0, 0, 0, 0)) + elif variant is SMPMainLayer: + for _ in range(4): + dpos += 1 + pixel_data.push_back(data_raw[dpos]) + + row_data.push_back(pixel(color_standard, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data.clear() + + elif lower_crumb == 0b00000010 and variant in (SMPMainLayer, SMPShadowLayer): + # player_color command + # draw the following 'count' pixels + # pixels are stored as 4 byte palette and meta infos + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + if variant is SMPShadowLayer: + dpos += 1 + nextbyte = data_raw[dpos] + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + else: + for _ in range(4): + dpos += 1 + pixel_data.push_back(data_raw[dpos]) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data.clear() + + else: + raise Exception( + f"unknown smp {variant.info.layer_type} layer drawing command: " + + f"{cmd:#x} in row {rowid:d}" + ) + + # process next command + dpos += 1 + + # end of row reached, return the created pixel array. + return + + cdef class SMPLayer: """ one layer inside the SMP. you can imagine it as a frame of a video. @@ -357,10 +521,11 @@ cdef class SMPLayer: row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) # process the drawing commands for this row. - self.process_drawing_cmds(data_raw, - row_data, rowid, - first_cmd_offset, - pixel_count - bounds.right) + process_drawing_cmds(self, + data_raw, + row_data, rowid, + first_cmd_offset, + pixel_count - bounds.right) # finish by filling up the right transparent space for i in range(bounds.right): @@ -383,15 +548,6 @@ cdef class SMPLayer: return row_data - @cython.boundscheck(False) - cdef void process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - size_t expected_size): - pass - def get_picture_data(self, palette): """ Convert the palette index matrix to a colored image. @@ -417,323 +573,6 @@ cdef class SMPLayer: return repr(self.info) -cdef class SMPMainLayer(SMPLayer): - """ - SMPLayer for the main graphics sprite. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef void process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands - found for this row in the SMP frame. - """ - - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd - cdef uint8_t nextbyte - cdef uint8_t lower_crumb - cdef int pixel_count - - cdef vector[uint8_t] pixel_data - pixel_data.reserve(4) - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d}" - ) - - # fetch drawing instruction - cmd = data_raw[dpos] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - # opcode: cmd, rowid: rowid - - if lower_crumb == 0b00000011: - # eol (end of line) command, this row is finished now. - eor = True - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # pixels are stored as 4 byte palette and meta infos - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - for _ in range(4): - dpos += 1 - pixel_data.push_back(data_raw[dpos]) - - row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here - - pixel_data.clear() - - elif lower_crumb == 0b00000010: - # player_color command - # draw the following 'count' pixels - # pixels are stored as 4 byte palette and meta infos - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - for _ in range(4): - dpos += 1 - pixel_data.push_back(data_raw[dpos]) - - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here - - pixel_data.clear() - - else: - raise Exception( - f"unknown smp main graphics layer drawing command: " + - f"{cmd:#x} in row {rowid:d}" - ) - - # process next command - dpos += 1 - - # end of row reached, return the created pixel array. - return - - def get_damage_mask(self): - """ - Convert the 4th pixel byte to a mask used for damaged units. - """ - return determine_damage_matrix(self.pcolor) - - -cdef class SMPShadowLayer(SMPLayer): - """ - SMPLayer for the shadow graphics. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef void process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands - found for this row in the SMP frame. - """ - - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd - cdef uint8_t nextbyte - cdef uint8_t lower_crumb - cdef int pixel_count - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - # opcode: cmd, rowid: rowid - - if lower_crumb == 0b00000011: - # eol (end of line) command, this row is finished now. - eor = True - - # shadows sometimes need an extra pixel at - # the end - if row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # pixels are stored as 1 byte alpha values - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - - dpos += 1 - nextbyte = data_raw[dpos] - - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - - else: - raise Exception( - f"unknown smp shadow layer drawing command: " + - f"{cmd:#x} in row {rowid:d}") - - # process next command - dpos += 1 - - # end of row reached, return the created pixel array. - return - - -cdef class SMPOutlineLayer(SMPLayer): - """ - SMPLayer for the outline graphics. - """ - - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - - @cython.boundscheck(False) - cdef void process_drawing_cmds(self, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - size_t expected_size): - """ - extract colors (pixels) for the drawing commands - found for this row in the SMP frame. - """ - - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd - cdef uint8_t nextbyte - cdef uint8_t lower_crumb - cdef int pixel_count - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - - # opcode: cmd, rowid: rowid - - if lower_crumb == 0b00000011: - # eol (end of line) command, this row is finished now. - eor = True - - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # as player outline colors. - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # we don't know the color the game wants - # so we just draw index 0 - row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) - - else: - raise Exception( - f"unknown smp outline layer drawing command: " + - f"{cmd:#x} in row {rowid:d}") - # process next command - dpos += 1 - - # end of row reached, return the created pixel array. - return @cython.boundscheck(False) From 536c6b8a30e862926709f6081822c19658f80f03 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Fri, 27 Dec 2024 22:42:44 +0000 Subject: [PATCH 06/17] formatting --- .../convert/value_object/read/media/smp.pyx | 71 +++++++++---------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 9454bd9c80..942ac8e028 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -1,7 +1,10 @@ -# Copyright 2013-2023 the openage authors. See copying.md for legal info. +# Copyright 2013-2024 the openage authors. See copying.md for legal info. # # cython: infer_types=True +from libcpp.vector cimport vector +from libcpp cimport bool +from libc.stdint cimport uint8_t, uint16_t from enum import Enum import numpy from struct import Struct, unpack_from @@ -12,11 +15,6 @@ from .....log import spam, dbg cimport cython cimport numpy -from libc.stdint cimport uint8_t, uint16_t -from libcpp cimport bool -from libcpp.vector cimport vector - - # SMP files have little endian byte order endianness = "< " @@ -50,8 +48,8 @@ class SMPLayerType(Enum): """ SMP layer types. """ - MAIN = "main" - SHADOW = "shadow" + MAIN = "main" + SHADOW = "shadow" OUTLINE = "outline" @@ -106,7 +104,7 @@ class SMP: def __init__(self, data): smp_header = SMP.smp_header.unpack_from(data) - signature, version, frame_count, facet_count, frames_per_facet,\ + signature, version, frame_count, facet_count, frames_per_facet, \ checksum, file_size, source_format, comment = smp_header dbg("SMP") @@ -159,12 +157,14 @@ class SMP: elif layer_header.layer_type == 0x04: # layer that stores a shadow - self.shadow_frames.append(SMPShadowLayer(layer_header, data)) + self.shadow_frames.append( + SMPShadowLayer(layer_header, data)) elif layer_header.layer_type == 0x08 or \ - layer_header.layer_type == 0x10: + layer_header.layer_type == 0x10: # layer that stores an outline - self.outline_frames.append(SMPOutlineLayer(layer_header, data)) + self.outline_frames.append( + SMPOutlineLayer(layer_header, data)) else: raise Exception( @@ -282,11 +282,11 @@ ctypedef fused SMPLayerVariant: @cython.boundscheck(False) cdef void process_drawing_cmds(SMPLayerVariant variant, - const uint8_t[::1] &data_raw, - vector[pixel] &row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - size_t expected_size): + const uint8_t[::1] & data_raw, + vector[pixel] & row_data, + Py_ssize_t rowid, + Py_ssize_t first_cmd_offset, + size_t expected_size): """ extract colors (pixels) for the drawing commands @@ -360,22 +360,22 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, dpos += 1 nextbyte = data_raw[dpos] row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) + nextbyte, 0, 0, 0)) elif variant is SMPOutlineLayer: # we don't know the color the game wants # so we just draw index 0 row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) + 0, 0, 0, 0)) elif variant is SMPMainLayer: for _ in range(4): dpos += 1 pixel_data.push_back(data_raw[dpos]) row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here pixel_data.clear() elif lower_crumb == 0b00000010 and variant in (SMPMainLayer, SMPShadowLayer): @@ -391,17 +391,17 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, dpos += 1 nextbyte = data_raw[dpos] row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) + nextbyte, 0, 0, 0)) else: for _ in range(4): dpos += 1 pixel_data.push_back(data_raw[dpos]) row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here pixel_data.clear() else: @@ -493,7 +493,7 @@ cdef class SMPLayer: self.pcolor.push_back(self.create_color_row(data_raw, i)) cdef vector[pixel] create_color_row(self, - const uint8_t[::1] &data_raw, + const uint8_t[::1] & data_raw, Py_ssize_t rowid): """ extract colors (pixels) for the given rowid. @@ -573,12 +573,10 @@ cdef class SMPLayer: return repr(self.info) - - @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, - numpy.ndarray[numpy.uint8_t, ndim=2, mode="c"] palette): +cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, + numpy.ndarray[numpy.uint8_t, ndim = 2, mode = "c"] palette): """ converts a palette index image matrix to an rgba matrix. """ @@ -586,7 +584,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t[:, ::1] m_lookup = palette @@ -681,9 +679,10 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, return array_data + @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): +cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix): """ converts the damage modifier values to an image using the RG values. @@ -693,7 +692,7 @@ cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t r From 11e2177984e99da3f97274934bf48a90fde89609 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Mon, 30 Dec 2024 02:11:25 +0000 Subject: [PATCH 07/17] adjustments --- .../convert/value_object/read/media/smp.pyx | 162 +++++++------ .../convert/value_object/read/media/smx.pyx | 222 ++++++++++-------- 2 files changed, 211 insertions(+), 173 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 942ac8e028..80d2844fce 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -258,6 +258,9 @@ cdef class SMPMainLayer(SMPLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + for i in range(self.row_count): + self.pcolor.push_back(create_color_row(self, i)) + def get_damage_mask(self): """ Convert the 4th pixel byte to a mask used for damaged units. @@ -268,18 +271,78 @@ cdef class SMPShadowLayer(SMPLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + for i in range(self.row_count): + self.pcolor.push_back(create_color_row(self, i)) + cdef class SMPOutlineLayer(SMPLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + for i in range(self.row_count): + self.pcolor.push_back(create_color_row(self, i)) + ctypedef fused SMPLayerVariant: - SMPLayer SMPMainLayer SMPShadowLayer SMPOutlineLayer +@cython.boundscheck(False) +cdef vector[pixel] create_color_row(SMPLayerVariant variant, + Py_ssize_t rowid): + """ + extract colors (pixels) for the given rowid. + """ + cdef vector[pixel] row_data + cdef Py_ssize_t i + + cdef Py_ssize_t first_cmd_offset = variant.cmd_offsets[rowid] # redudent + cdef boundary_def bounds = variant.boundaries[rowid] + cdef size_t pixel_count = variant.info.size[0] + + # preallocate memory + row_data.reserve(pixel_count) + + # row is completely transparent + if bounds.full_row: + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + return row_data + + # start drawing the left transparent space + for i in range(bounds.left): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + # process the drawing commands for this row. + process_drawing_cmds(variant, variant.data_raw, + row_data, rowid, + first_cmd_offset, + pixel_count - bounds.right) + + # finish by filling up the right transparent space + for i in range(bounds.right): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + # verify size of generated row + if row_data.size() != pixel_count: + got = row_data.size() + summary = ( + f"{got:d}/{pixel_count:d} -> row {rowid:d}, " + f"layer type {variant.info.layer_type:x}, " + f"offset {first_cmd_offset:d} / {first_cmd_offset:#x}" + ) + message = ( + f"got {'LESS' if got < pixel_count else 'MORE'} pixels than expected: {summary}, " + f"missing: {abs(pixel_count - got):d}" + ) + + raise Exception(message) + + return row_data + + @cython.boundscheck(False) cdef void process_drawing_cmds(SMPLayerVariant variant, const uint8_t[::1] & data_raw, @@ -312,7 +375,7 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, if row_data.size() > expected_size: raise Exception( f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {variant.info.layer_type:#x}, but we have {row_data.size():d}" + f"with layer type {variant.info.layer_type:#x}, but we have {row_data.size():d} already!" ) # fetch drawing instruction @@ -320,14 +383,13 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, # Last 2 bits store command type lower_crumb = 0b00000011 & cmd - # opcode: cmd, rowid: rowid if lower_crumb == 0b00000011: # eol (end of line) command, this row is finished now. eor = True - if variant is SMPShadowLayer and row_data.size() < expected_size: + if SMPLayerVariant is SMPShadowLayer and row_data.size() < expected_size: # copy the last drawn pixel # (still stored in nextbyte) # @@ -356,17 +418,17 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, pixel_count = (cmd >> 2) + 1 for _ in range(pixel_count): - if variant is SMPShadowLayer: + if SMPLayerVariant is SMPShadowLayer: dpos += 1 nextbyte = data_raw[dpos] row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) - elif variant is SMPOutlineLayer: + elif SMPLayerVariant is SMPOutlineLayer: # we don't know the color the game wants # so we just draw index 0 row_data.push_back(pixel(color_outline, 0, 0, 0, 0)) - elif variant is SMPMainLayer: + elif SMPLayerVariant is SMPMainLayer: for _ in range(4): dpos += 1 pixel_data.push_back(data_raw[dpos]) @@ -378,7 +440,7 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, pixel_data[3] & 0x1F)) # remove "usage" bit here pixel_data.clear() - elif lower_crumb == 0b00000010 and variant in (SMPMainLayer, SMPShadowLayer): + elif lower_crumb == 0b00000010 and (SMPLayerVariant is SMPMainLayer or SMPLayerVariant is SMPShadowLayer): # player_color command # draw the following 'count' pixels # pixels are stored as 4 byte palette and meta infos @@ -387,7 +449,7 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, pixel_count = (cmd >> 2) + 1 for _ in range(pixel_count): - if variant is SMPShadowLayer: + if SMPLayerVariant is SMPShadowLayer: dpos += 1 nextbyte = data_raw[dpos] row_data.push_back(pixel(color_shadow, @@ -446,15 +508,20 @@ cdef class SMPLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor + # memory pointer + cdef const uint8_t[::1] data_raw + + # rows of image + cdef size_t row_count + def __init__(self, layer_header, data): self.info = layer_header if not (isinstance(data, bytes) or isinstance(data, bytearray)): raise ValueError("Layer data must be some bytes object") - # memory pointer # convert the bytes obj to char* - cdef const uint8_t[::1] data_raw = data + self.data_raw = data cdef unsigned short left cdef unsigned short right @@ -462,11 +529,11 @@ cdef class SMPLayer: cdef size_t i cdef int cmd_offset - cdef size_t row_count = self.info.size[1] - self.pcolor.reserve(row_count) + self.row_count = self.info.size[1] + self.pcolor.reserve(self.row_count) # process bondary table - for i in range(row_count): + for i in range(self.row_count): outline_entry_position = (self.info.outline_table_offset + i * SMPLayer.smp_frame_row_edge.size) @@ -481,7 +548,7 @@ cdef class SMPLayer: self.boundaries.push_back(boundary_def(left, right, False)) # process cmd table - for i in range(row_count): + for i in range(self.row_count): cmd_table_position = (self.info.qdl_table_offset + i * SMPLayer.smp_command_offset.size) @@ -489,65 +556,6 @@ cdef class SMPLayer: data, cmd_table_position)[0] + self.info.frame_offset self.cmd_offsets.push_back(cmd_offset) - for i in range(row_count): - self.pcolor.push_back(self.create_color_row(data_raw, i)) - - cdef vector[pixel] create_color_row(self, - const uint8_t[::1] & data_raw, - Py_ssize_t rowid): - """ - extract colors (pixels) for the given rowid. - """ - - cdef vector[pixel] row_data - cdef Py_ssize_t i - - first_cmd_offset = self.cmd_offsets[rowid] - cdef boundary_def bounds = self.boundaries[rowid] - cdef size_t pixel_count = self.info.size[0] - - # preallocate memory - row_data.reserve(pixel_count) - - # row is completely transparent - if bounds.full_row: - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - return row_data - - # start drawing the left transparent space - for i in range(bounds.left): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # process the drawing commands for this row. - process_drawing_cmds(self, - data_raw, - row_data, rowid, - first_cmd_offset, - pixel_count - bounds.right) - - # finish by filling up the right transparent space - for i in range(bounds.right): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # verify size of generated row - if row_data.size() != pixel_count: - got = row_data.size() - summary = ( - f"{got:d}/{pixel_count:d} -> row {rowid:d}, " - f"layer type {self.info.layer_type:x}, " - f"offset {first_cmd_offset:d} / {first_cmd_offset:#x}" - ) - message = ( - f"got {'LESS' if got < pixel_count else 'MORE'} pixels than expected: {summary}, " - f"missing: {abs(pixel_count - got):d}" - ) - - raise Exception(message) - - return row_data - def get_picture_data(self, palette): """ Convert the palette index matrix to a colored image. @@ -576,7 +584,7 @@ cdef class SMPLayer: @cython.boundscheck(False) @cython.wraparound(False) cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, - numpy.ndarray[numpy.uint8_t, ndim = 2, mode = "c"] palette): + numpy.ndarray[numpy.uint8_t, ndim= 2, mode = "c"] palette): """ converts a palette index image matrix to an rgba matrix. """ @@ -584,7 +592,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t[:, ::1] m_lookup = palette @@ -692,7 +700,7 @@ cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix) cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t r diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index 2cb4749091..fd12a20ee7 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -60,28 +60,116 @@ cdef public dict LAYER_TYPES = { cdef class SMXMainLayer8to5(SMXLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + + self.pcolor.push_back(row_data) cdef class SMXMainLayer4plus1(SMXLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + + self.pcolor.push_back(row_data) cdef class SMXOutlineLayer(SMXLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + + self.pcolor.push_back(row_data) cdef class SMXShadowLayer(SMXLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + + self.pcolor.push_back(row_data) ctypedef fused SMXLayerVariant: - SMXLayer SMXMainLayer8to5 SMXMainLayer4plus1 SMXOutlineLayer SMXShadowLayer +cdef inline(int, int, int, vector[pixel]) create_color_row(SMXLayerVariant variant, + Py_ssize_t rowid): + """ + Extract colors (pixels) for the given rowid. + + :param rowid: Index of the current row in the layer. + :param cmd_offset: Offset of the command table of the layer. + :param color_offset: Offset of the color table of the layer. + :param chunk_pos: Current position in the compressed chunk. + """ + + cdef vector[pixel] row_data + cdef Py_ssize_t i + + cdef int first_cmd_offset = variant.cmd_offset + cdef int first_color_offset = variant.color_offset + cdef int first_chunk_pos = variant.chunk_pos + cdef boundary_def bounds = variant.boundaries[rowid] + cdef size_t pixel_count = variant.info.size[0] + + # preallocate memory + row_data.reserve(pixel_count) + + # row is completely transparent + if bounds.full_row: + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + return variant.cmd_offset, variant.color_offset, variant.chunk_pos, row_data + + # start drawing the left transparent space + for i in range(bounds.left): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + # process the drawing commands for this row. + next_cmd_offset, next_color_offset, next_chunk_pos, row_data = \ + process_drawing_cmds(variant, + variant.data_raw, + row_data, + rowid, + first_cmd_offset, + first_color_offset, + first_chunk_pos, + pixel_count - bounds.right + ) + + # finish by filling up the right transparent space + for i in range(bounds.right): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + # verify size of generated row + if row_data.size() != pixel_count: + got = row_data.size() + summary = "%d/%d -> row %d, layer type %s, offset %d / %#x" % ( + got, pixel_count, rowid, variant.info.layer_type, + first_cmd_offset, first_cmd_offset + ) + txt = "got %%s pixels than expected: %s, missing: %d" % ( + summary, abs(pixel_count - got)) + + raise Exception(txt % ("LESS" if got < pixel_count else "MORE")) + + return next_cmd_offset, next_color_offset, next_chunk_pos, row_data + + @cython.boundscheck(False) cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant variant, const uint8_t[::1] & data_raw, @@ -117,7 +205,9 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v cdef uint8_t pixel_mask_even_2 = 0b11110000 cdef uint8_t pixel_mask_even_3 = 0b00111111 - if variant in (SMXMainLayer8to5, SMXMainLayer4plus1): + print(type(variant)) + + if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: # Position in the pixel data array dpos_color = first_color_offset @@ -139,13 +229,14 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v # Last 2 bits store command type lower_crumb = 0b00000011 & cmd + print(lower_crumb) if lower_crumb == 0b00000011: # eor (end of row) command, this row is finished now. eor = True dpos_cmd += 1 - if variant is SMXShadowLayer: + if SMXLayerVariant is SMXShadowLayer: # shadows sometimes need an extra pixel at # the end if row_data.size() < expected_size: @@ -179,7 +270,7 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v pixel_count = (cmd >> 2) + 1 - if variant is SMXMainLayer8to5: + if SMXLayerVariant is SMXMainLayer8to5: pixel_data.reserve(4) for _ in range(pixel_count): # Start fetching pixel data @@ -229,7 +320,7 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v odd = not odd pixel_data.clear() - if variant is SMXMainLayer4plus1: + if SMXLayerVariant is SMXMainLayer4plus1: palette_section_block = data_raw[dpos_color + (4 - chunk_pos)] for _ in range(pixel_count): @@ -251,7 +342,7 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] - if variant is SMXShadowLayer: + if SMXLayerVariant is SMXShadowLayer: for _ in range(pixel_count): dpos_color += 1 nextbyte = data_raw[dpos_color] @@ -259,14 +350,14 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) - if variant is SMXOutlineLayer: + if SMXLayerVariant is SMXOutlineLayer: # we don't know the color the game wants # so we just draw index 0 row_data.push_back(pixel(color_outline, 0, 0, 0, 0)) elif lower_crumb == 0b00000010: - if variant is SMXMainLayer8to5: + if SMXLayerVariant is SMXMainLayer8to5: # player_color command # draw the following 'count' pixels # pixels are stored in 5 byte chunks @@ -327,7 +418,7 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v pixel_data.clear() elif lower_crumb == 0b00000010: - if variant is SMXMainLayer4plus1: + if SMXLayerVariant is SMXMainLayer4plus1: # player_color command # draw the following 'count' pixels # 4 pixels are stored in every 5 byte chunk. @@ -356,7 +447,7 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] - if variant is (SMXOutlineLayer, SMXShadowLayer): + if SMXLayerVariant is SMXOutlineLayer or SMXLayerVariant is SMXShadowLayer: pass else: @@ -368,9 +459,9 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v # Process next command dpos_cmd += 1 - if variant in (SMXMainLayer8to5, SMXMainLayer4plus1): - return dpos_cmd, dpos_color, odd, row_data - if variant in (SMXOutlineLayer, SMXShadowLayer): + if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: + return dpos_cmd, dpos_color, chunk_pos, row_data + if SMXLayerVariant is SMXOutlineLayer or SMXLayerVariant is SMXShadowLayer: return dpos_cmd, dpos_cmd, chunk_pos, row_data @@ -640,6 +731,21 @@ cdef class SMXLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor + # memory pointer + cdef const uint8_t[::1] data_raw + + # current command + cdef int cmd_offset + + # current color + cdef int color_offset + + # current chunk position + cdef int chunk_pos + + # rows + cdef size_t row_count + def __init__(self, layer_header, data): """ SMX layer definition superclass. There can be various types of @@ -655,19 +761,18 @@ cdef class SMXLayer: if not (isinstance(data, bytes) or isinstance(data, bytearray)): raise ValueError("Layer data must be some bytes object") - # memory pointer # convert the bytes obj to char* - cdef const uint8_t[::1] data_raw = data + self.data_raw = data cdef unsigned short left cdef unsigned short right cdef size_t i - cdef size_t row_count = self.info.size[1] - self.pcolor.reserve(row_count) + self.row_count = self.info.size[1] + self.pcolor.reserve(self.row_count) # process bondary table - for i in range(row_count): + for i in range(self.row_count): outline_entry_position = (self.info.outline_table_offset + i * SMXLayer.smp_layer_row_edge.size) @@ -681,84 +786,9 @@ cdef class SMXLayer: else: self.boundaries.push_back(boundary_def(left, right, False)) - cdef int cmd_offset = self.info.qdl_command_table_offset - cdef int color_offset = self.info.qdl_color_table_offset - cdef int chunk_pos = 0 - - # process cmd table - for i in range(row_count): - cmd_offset, color_offset, chunk_pos, row_data = \ - self.create_color_row( - data_raw, i, cmd_offset, color_offset, chunk_pos) - - self.pcolor.push_back(row_data) - - cdef inline(int, int, int, vector[pixel]) create_color_row(self, - const uint8_t[::1] & data_raw, - Py_ssize_t rowid, - int cmd_offset, - int color_offset, - int chunk_pos): - """ - Extract colors (pixels) for the given rowid. - - :param rowid: Index of the current row in the layer. - :param cmd_offset: Offset of the command table of the layer. - :param color_offset: Offset of the color table of the layer. - :param chunk_pos: Current position in the compressed chunk. - """ - - cdef vector[pixel] row_data - cdef Py_ssize_t i - - cdef int first_cmd_offset = cmd_offset - cdef int first_color_offset = color_offset - cdef boundary_def bounds = self.boundaries[rowid] - cdef size_t pixel_count = self.info.size[0] - - # preallocate memory - row_data.reserve(pixel_count) - - # row is completely transparent - if bounds.full_row: - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - return cmd_offset, color_offset, chunk_pos, row_data - - # start drawing the left transparent space - for i in range(bounds.left): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # process the drawing commands for this row. - next_cmd_offset, next_color_offset, chunk_pos, row_data = \ - process_drawing_cmds(self, - data_raw, - row_data, - rowid, - first_cmd_offset, - first_color_offset, - chunk_pos, - pixel_count - bounds.right - ) - - # finish by filling up the right transparent space - for i in range(bounds.right): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # verify size of generated row - if row_data.size() != pixel_count: - got = row_data.size() - summary = "%d/%d -> row %d, layer type %d, offset %d / %#x" % ( - got, pixel_count, rowid, self.info.layer_type, - first_cmd_offset, first_cmd_offset - ) - txt = "got %%s pixels than expected: %s, missing: %d" % ( - summary, abs(pixel_count - got)) - - raise Exception(txt % ("LESS" if got < pixel_count else "MORE")) - - return next_cmd_offset, next_color_offset, chunk_pos, row_data + self.cmd_offset = self.info.qdl_command_table_offset + self.color_offset = self.info.qdl_color_table_offset + self.chunk_pos = 0 def get_picture_data(self, palette): """ From 21af7181cb4593031f067d7861f7d8ceac6969b4 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Mon, 30 Dec 2024 04:20:01 +0000 Subject: [PATCH 08/17] adjustments --- .../convert/value_object/read/media/smp.pyx | 2 +- .../convert/value_object/read/media/smx.pyx | 57 ++++++++++--------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 80d2844fce..15cb55873e 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -297,7 +297,7 @@ cdef vector[pixel] create_color_row(SMPLayerVariant variant, cdef vector[pixel] row_data cdef Py_ssize_t i - cdef Py_ssize_t first_cmd_offset = variant.cmd_offsets[rowid] # redudent + cdef Py_ssize_t first_cmd_offset = variant.cmd_offsets[rowid] cdef boundary_def bounds = variant.boundaries[rowid] cdef size_t pixel_count = variant.info.size[0] diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index fd12a20ee7..a058c22399 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -1,4 +1,4 @@ -# Copyright 2019-2023 the openage authors. See copying.md for legal info. +# Copyright 2019-2024 the openage authors. See copying.md for legal info. # # cython: infer_types=True @@ -64,6 +64,9 @@ cdef class SMXMainLayer8to5(SMXLayer): for i in range(self.row_count): cmd_offset, color_offset, chunk_pos, row_data = create_color_row( self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos self.pcolor.push_back(row_data) @@ -74,6 +77,9 @@ cdef class SMXMainLayer4plus1(SMXLayer): for i in range(self.row_count): cmd_offset, color_offset, chunk_pos, row_data = create_color_row( self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos self.pcolor.push_back(row_data) @@ -84,6 +90,9 @@ cdef class SMXOutlineLayer(SMXLayer): for i in range(self.row_count): cmd_offset, color_offset, chunk_pos, row_data = create_color_row( self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos self.pcolor.push_back(row_data) @@ -94,6 +103,9 @@ cdef class SMXShadowLayer(SMXLayer): for i in range(self.row_count): cmd_offset, color_offset, chunk_pos, row_data = create_color_row( self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos self.pcolor.push_back(row_data) @@ -182,7 +194,6 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v """ TODO: docstring """ - # position in the command array, we start at the first command of this row cdef Py_ssize_t dpos_cmd = first_cmd_offset @@ -205,8 +216,6 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v cdef uint8_t pixel_mask_even_2 = 0b11110000 cdef uint8_t pixel_mask_even_3 = 0b00111111 - print(type(variant)) - if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: # Position in the pixel data array dpos_color = first_color_offset @@ -229,24 +238,22 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v # Last 2 bits store command type lower_crumb = 0b00000011 & cmd - print(lower_crumb) - if lower_crumb == 0b00000011: # eor (end of row) command, this row is finished now. + eor = True dpos_cmd += 1 - if SMXLayerVariant is SMXShadowLayer: - # shadows sometimes need an extra pixel at - # the end - if row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) + # shadows sometimes need an extra pixel at + # the end + if SMXLayerVariant is SMXShadowLayer and row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) continue elif lower_crumb == 0b00000000: @@ -344,8 +351,8 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v if SMXLayerVariant is SMXShadowLayer: for _ in range(pixel_count): - dpos_color += 1 - nextbyte = data_raw[dpos_color] + dpos_cmd += 1 + nextbyte = data_raw[dpos_cmd] row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) @@ -417,8 +424,7 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v odd = not odd pixel_data.clear() - elif lower_crumb == 0b00000010: - if SMXLayerVariant is SMXMainLayer4plus1: + elif SMXLayerVariant is SMXMainLayer4plus1: # player_color command # draw the following 'count' pixels # 4 pixels are stored in every 5 byte chunk. @@ -447,9 +453,6 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v dpos_color += 1 # Skip palette section block palette_section_block = data_raw[dpos_color + 4] - if SMXLayerVariant is SMXOutlineLayer or SMXLayerVariant is SMXShadowLayer: - pass - else: raise Exception( f"unknown smx main graphics layer drawing command: " + @@ -826,7 +829,7 @@ cdef class SMXLayer: @cython.boundscheck(False) @cython.wraparound(False) cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, - numpy.ndarray[numpy.uint8_t, ndim = 2, mode = "c"] palette): + numpy.ndarray[numpy.uint8_t, ndim= 2, mode = "c"] palette): """ converts a palette index image matrix to an rgba matrix. @@ -837,7 +840,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t[:, ::1] m_lookup = palette @@ -936,7 +939,7 @@ cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix) cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim = 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t r = 0 From f8d35e074179b561245898596cb4620de09a2e87 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Thu, 2 Jan 2025 16:19:12 +0000 Subject: [PATCH 09/17] copying.md --- .mailmap | 6 +++++- copying.md | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 8a1a8727b1..3d8b8ec12e 100644 --- a/.mailmap +++ b/.mailmap @@ -21,4 +21,8 @@ Tobias Feldballe Jonas Borchelt Derek Frogget <114030121+derekfrogget@users.noreply.github.com> Nikhil Ghosh -David Wever <56411717+dmwever@users.noreply.github.com> \ No newline at end of file +<<<<<<< HEAD +David Wever <56411717+dmwever@users.noreply.github.com> +======= +Ngô Xuân Minh +>>>>>>> e8251ff6 (copying.md) diff --git a/copying.md b/copying.md index 53ced52632..f873817f29 100644 --- a/copying.md +++ b/copying.md @@ -160,6 +160,7 @@ _the openage authors_ are: | Alex Zhuohao He | ZzzhHe | zhuohao dawt he à outlook dawt com | | David Wever | dmwever | dmwever à crimson dawt ua dawt edu | | Michael Lynch | mtlynch | git à mtlynch dawt io | +| Ngô Xuân Minh | | xminh dawt ngo dawt 00 à gmail dawt com | If you're a first-time committer, add yourself to the above list. This is not just for legal reasons, but also to keep an overview of all those nicknames. From e6228a2e4326c5654e7968b74c1c4142da91e99b Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Thu, 9 Jan 2025 13:08:08 +0000 Subject: [PATCH 10/17] addressing comments --- .../convert/value_object/read/media/smp.pyx | 235 +++++++++--------- .../convert/value_object/read/media/smx.pyx | 182 +++++++------- 2 files changed, 211 insertions(+), 206 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 15cb55873e..c52aa1a194 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -1,10 +1,7 @@ -# Copyright 2013-2024 the openage authors. See copying.md for legal info. +# Copyright 2013-2025 the openage authors. See copying.md for legal info. # # cython: infer_types=True -from libcpp.vector cimport vector -from libcpp cimport bool -from libc.stdint cimport uint8_t, uint16_t from enum import Enum import numpy from struct import Struct, unpack_from @@ -15,6 +12,11 @@ from .....log import spam, dbg cimport cython cimport numpy +from libc.stdint cimport uint8_t, uint16_t +from libcpp cimport bool +from libcpp.vector cimport vector + + # SMP files have little endian byte order endianness = "< " @@ -48,8 +50,8 @@ class SMPLayerType(Enum): """ SMP layer types. """ - MAIN = "main" - SHADOW = "shadow" + MAIN = "main" + SHADOW = "shadow" OUTLINE = "outline" @@ -104,7 +106,7 @@ class SMP: def __init__(self, data): smp_header = SMP.smp_header.unpack_from(data) - signature, version, frame_count, facet_count, frames_per_facet, \ + signature, version, frame_count, facet_count, frames_per_facet,\ checksum, file_size, source_format, comment = smp_header dbg("SMP") @@ -157,14 +159,12 @@ class SMP: elif layer_header.layer_type == 0x04: # layer that stores a shadow - self.shadow_frames.append( - SMPShadowLayer(layer_header, data)) + self.shadow_frames.append(SMPShadowLayer(layer_header, data)) elif layer_header.layer_type == 0x08 or \ - layer_header.layer_type == 0x10: + layer_header.layer_type == 0x10: # layer that stores an outline - self.outline_frames.append( - SMPOutlineLayer(layer_header, data)) + self.outline_frames.append(SMPOutlineLayer(layer_header, data)) else: raise Exception( @@ -254,6 +254,108 @@ class SMPLayerHeader: return "".join(ret) +cdef class SMPLayer: + """ + one layer inside the SMP. you can imagine it as a frame of a video. + """ + + # struct smp_frame_row_edge { + # unsigned short left_space; + # unsigned short right_space; + # }; + smp_frame_row_edge = Struct(endianness + "H H") + + # struct smp_command_offset { + # unsigned int offset; + # } + smp_command_offset = Struct(endianness + "I") + + # layer and frame information + cdef object info + + # for each row: + # contains (left, right, full_row) number of boundary pixels + cdef vector[boundary_def] boundaries + + # stores the file offset for the first drawing command + cdef vector[int] cmd_offsets + + # pixel matrix representing the final image + cdef vector[vector[pixel]] pcolor + + # memory pointer + cdef const uint8_t[::1] data_raw + + # rows of image + cdef size_t row_count + + def __init__(self, layer_header, data): + self.info = layer_header + + if not (isinstance(data, bytes) or isinstance(data, bytearray)): + raise ValueError("Layer data must be some bytes object") + + # convert the bytes obj to char* + self.data_raw = data + + cdef unsigned short left + cdef unsigned short right + + cdef size_t i + cdef int cmd_offset + + self.row_count = self.info.size[1] + self.pcolor.reserve(self.row_count) + + # process bondary table + for i in range(self.row_count): + outline_entry_position = (self.info.outline_table_offset + + i * SMPLayer.smp_frame_row_edge.size) + + left, right = SMPLayer.smp_frame_row_edge.unpack_from( + data, outline_entry_position + ) + + # is this row completely transparent? + if left == 0xFFFF or right == 0xFFFF: + self.boundaries.push_back(boundary_def(0, 0, True)) + else: + self.boundaries.push_back(boundary_def(left, right, False)) + + # process cmd table + for i in range(self.row_count): + cmd_table_position = (self.info.qdl_table_offset + + i * SMPLayer.smp_command_offset.size) + + cmd_offset = SMPLayer.smp_command_offset.unpack_from( + data, cmd_table_position)[0] + self.info.frame_offset + self.cmd_offsets.push_back(cmd_offset) + + def get_picture_data(self, palette): + """ + Convert the palette index matrix to a colored image. + """ + return determine_rgba_matrix(self.pcolor, palette) + + def get_hotspot(self): + """ + Return the layer's hotspot (the "center" of the image) + """ + return self.info.hotspot + + def get_palette_number(self): + """ + Return the layer's palette number. + + :return: Palette number of the layer. + :rtype: int + """ + return self.pcolor[0][0].palette & 0b00111111 + + def __repr__(self): + return repr(self.info) + + cdef class SMPMainLayer(SMPLayer): def __init__(self, layer_header, data): super().__init__(layer_header, data) @@ -479,112 +581,13 @@ cdef void process_drawing_cmds(SMPLayerVariant variant, return -cdef class SMPLayer: - """ - one layer inside the SMP. you can imagine it as a frame of a video. - """ - - # struct smp_frame_row_edge { - # unsigned short left_space; - # unsigned short right_space; - # }; - smp_frame_row_edge = Struct(endianness + "H H") - - # struct smp_command_offset { - # unsigned int offset; - # } - smp_command_offset = Struct(endianness + "I") - - # layer and frame information - cdef object info - - # for each row: - # contains (left, right, full_row) number of boundary pixels - cdef vector[boundary_def] boundaries - - # stores the file offset for the first drawing command - cdef vector[int] cmd_offsets - - # pixel matrix representing the final image - cdef vector[vector[pixel]] pcolor - - # memory pointer - cdef const uint8_t[::1] data_raw - - # rows of image - cdef size_t row_count - - def __init__(self, layer_header, data): - self.info = layer_header - - if not (isinstance(data, bytes) or isinstance(data, bytearray)): - raise ValueError("Layer data must be some bytes object") - - # convert the bytes obj to char* - self.data_raw = data - - cdef unsigned short left - cdef unsigned short right - - cdef size_t i - cdef int cmd_offset - - self.row_count = self.info.size[1] - self.pcolor.reserve(self.row_count) - - # process bondary table - for i in range(self.row_count): - outline_entry_position = (self.info.outline_table_offset + - i * SMPLayer.smp_frame_row_edge.size) - - left, right = SMPLayer.smp_frame_row_edge.unpack_from( - data, outline_entry_position - ) - - # is this row completely transparent? - if left == 0xFFFF or right == 0xFFFF: - self.boundaries.push_back(boundary_def(0, 0, True)) - else: - self.boundaries.push_back(boundary_def(left, right, False)) - - # process cmd table - for i in range(self.row_count): - cmd_table_position = (self.info.qdl_table_offset + - i * SMPLayer.smp_command_offset.size) - - cmd_offset = SMPLayer.smp_command_offset.unpack_from( - data, cmd_table_position)[0] + self.info.frame_offset - self.cmd_offsets.push_back(cmd_offset) - - def get_picture_data(self, palette): - """ - Convert the palette index matrix to a colored image. - """ - return determine_rgba_matrix(self.pcolor, palette) - - def get_hotspot(self): - """ - Return the layer's hotspot (the "center" of the image) - """ - return self.info.hotspot - - def get_palette_number(self): - """ - Return the layer's palette number. - - :return: Palette number of the layer. - :rtype: int - """ - return self.pcolor[0][0].palette & 0b00111111 - def __repr__(self): - return repr(self.info) @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, - numpy.ndarray[numpy.uint8_t, ndim= 2, mode = "c"] palette): +cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, + numpy.ndarray[numpy.uint8_t, ndim=2, mode="c"] palette): """ converts a palette index image matrix to an rgba matrix. """ @@ -592,7 +595,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t[:, ::1] m_lookup = palette @@ -690,7 +693,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix): +cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): """ converts the damage modifier values to an image using the RG values. @@ -700,7 +703,7 @@ cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix) cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t r diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index a058c22399..52e17428c8 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -1,10 +1,7 @@ -# Copyright 2019-2024 the openage authors. See copying.md for legal info. +# Copyright 2019-2025 the openage authors. See copying.md for legal info. # # cython: infer_types=True -from libcpp.vector cimport vector -from libcpp cimport bool -from libc.stdint cimport uint8_t, uint16_t from enum import Enum import numpy from struct import Struct, unpack_from @@ -15,6 +12,11 @@ from .....log import spam, dbg cimport cython cimport numpy +from libc.stdint cimport uint8_t, uint16_t +from libcpp cimport bool +from libcpp.vector cimport vector + + # SMX files have little endian byte order endianness = "< " @@ -47,8 +49,8 @@ class SMXLayerType(Enum): """ SMX layer types. """ - MAIN = "main" - SHADOW = "shadow" + MAIN = "main" + SHADOW = "shadow" OUTLINE = "outline" @@ -57,57 +59,6 @@ cdef public dict LAYER_TYPES = { 1: SMXLayerType.SHADOW, 2: SMXLayerType.OUTLINE, } -cdef class SMXMainLayer8to5(SMXLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos - - self.pcolor.push_back(row_data) - -cdef class SMXMainLayer4plus1(SMXLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos - - self.pcolor.push_back(row_data) - -cdef class SMXOutlineLayer(SMXLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos - - self.pcolor.push_back(row_data) - -cdef class SMXShadowLayer(SMXLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos - - self.pcolor.push_back(row_data) ctypedef fused SMXLayerVariant: @@ -246,15 +197,16 @@ cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant v # shadows sometimes need an extra pixel at # the end - if SMXLayerVariant is SMXShadowLayer and row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - continue + if SMXLayerVariant is SMXShadowLayer: + if row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + continue elif lower_crumb == 0b00000000: # skip command @@ -510,16 +462,14 @@ class SMX: """ smx_header = SMX.smx_header.unpack_from(data) - self.smp_type, version, frame_count, file_size_comp, \ + self.smp_type, version, frame_count, file_size_comp,\ file_size_uncomp, comment = smx_header dbg("SMX") dbg(" version: %s", version) dbg(" frame count: %s", frame_count) - # 0x20 = SMX header size - dbg(" file size compressed: %s B", file_size_comp + 0x20) - # 0x80 = SMP header size - dbg(" file size uncompressed: %s B", file_size_uncomp + 0x40) + dbg(" file size compressed: %s B", file_size_comp + 0x20) # 0x20 = SMX header size + dbg(" file size uncompressed: %s B", file_size_uncomp + 0x40) # 0x80 = SMP header size dbg(" comment: %s", comment.decode('ascii')) # SMX graphic frames are created from overlaying @@ -537,7 +487,7 @@ class SMX: frame_header = SMX.smx_frame_header.unpack_from( data, current_offset) - frame_type, palette_number, _ = frame_header + frame_type , palette_number, _ = frame_header current_offset += SMX.smx_frame_header.size @@ -556,7 +506,7 @@ class SMX: layer_header_data = SMX.smx_layer_header.unpack_from( data, current_offset) - width, height, hotspot_x, hotspot_y, \ + width, height, hotspot_x, hotspot_y,\ distance_next_frame, _ = layer_header_data current_offset += SMX.smx_layer_header.size @@ -566,14 +516,12 @@ class SMX: # Skip outline table current_offset += 4 * height - qdl_command_array_size = Struct( - "< I").unpack_from(data, current_offset)[0] + qdl_command_array_size = Struct("< I").unpack_from(data, current_offset)[0] current_offset += 4 # Read length of color table if layer_type is SMXLayerType.MAIN: - qdl_color_table_size = Struct( - "< I").unpack_from(data, current_offset)[0] + qdl_color_table_size = Struct("< I").unpack_from(data, current_offset)[0] current_offset += 4 qdl_color_table_offset = current_offset + qdl_command_array_size @@ -593,20 +541,16 @@ class SMX: if layer_type is SMXLayerType.MAIN: if layer_header.compression_type == 0x08: - self.main_frames.append( - SMXMainLayer8to5(layer_header, data)) + self.main_frames.append(SMXMainLayer8to5(layer_header, data)) elif layer_header.compression_type == 0x00: - self.main_frames.append( - SMXMainLayer4plus1(layer_header, data)) + self.main_frames.append(SMXMainLayer4plus1(layer_header, data)) elif layer_type is SMXLayerType.SHADOW: - self.shadow_frames.append( - SMXShadowLayer(layer_header, data)) + self.shadow_frames.append(SMXShadowLayer(layer_header, data)) elif layer_type is SMXLayerType.OUTLINE: - self.outline_frames.append( - SMXOutlineLayer(layer_header, data)) + self.outline_frames.append(SMXOutlineLayer(layer_header, data)) def get_frames(self, layer: int = 0): """ @@ -826,10 +770,68 @@ cdef class SMXLayer: return repr(self.info) +cdef class SMXMainLayer8to5(SMXLayer): + """ + Compressed SMP layer (compression type 8to5) for the main graphics sprite. + """ + + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos + + self.pcolor.push_back(row_data) + +cdef class SMXMainLayer4plus1(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos + + self.pcolor.push_back(row_data) + +cdef class SMXOutlineLayer(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos + + self.pcolor.push_back(row_data) + +cdef class SMXShadowLayer(SMXLayer): + def __init__(self, layer_header, data): + super().__init__(layer_header, data) + # process cmd table + for i in range(self.row_count): + cmd_offset, color_offset, chunk_pos, row_data = create_color_row( + self, i) + self.cmd_offset = cmd_offset + self.color_offset = color_offset + self.chunk_pos = chunk_pos + + self.pcolor.push_back(row_data) + + + @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, - numpy.ndarray[numpy.uint8_t, ndim= 2, mode = "c"] palette): +cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, + numpy.ndarray[numpy.uint8_t, ndim=2, mode="c"] palette): """ converts a palette index image matrix to an rgba matrix. @@ -840,7 +842,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t[:, ::1] m_lookup = palette @@ -929,7 +931,7 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] & image_matrix, @cython.boundscheck(False) @cython.wraparound(False) -cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix): +cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): """ converts the damage modifier values to an image using the RG values. @@ -939,7 +941,7 @@ cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] & image_matrix) cdef size_t height = image_matrix.size() cdef size_t width = image_matrix[0].size() - cdef numpy.ndarray[numpy.uint8_t, ndim= 3, mode = "c"] array_data = \ + cdef numpy.ndarray[numpy.uint8_t, ndim=3, mode="c"] array_data = \ numpy.zeros((height, width, 4), dtype=numpy.uint8) cdef uint8_t r = 0 From c8b98a2719f5198cf9d678dbd2d95921a6528069 Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Thu, 16 Jan 2025 23:26:07 +0000 Subject: [PATCH 11/17] restructure --- .../convert/value_object/read/media/smp.pyx | 423 +++++----- .../convert/value_object/read/media/smx.pyx | 794 +++++++++--------- 2 files changed, 592 insertions(+), 625 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index c52aa1a194..9ad253c5c3 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -253,6 +253,11 @@ class SMPLayerHeader: ) return "".join(ret) +ctypedef fused SMPLayerVariant: + SMPMainLayer + SMPShadowLayer + SMPOutlineLayer + cdef class SMPLayer: """ @@ -283,20 +288,15 @@ cdef class SMPLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor - # memory pointer - cdef const uint8_t[::1] data_raw - - # rows of image - cdef size_t row_count - - def __init__(self, layer_header, data): + def init(self, SMPLayerVariant variant, layer_header, data): self.info = layer_header if not (isinstance(data, bytes) or isinstance(data, bytearray)): raise ValueError("Layer data must be some bytes object") + # memory pointer # convert the bytes obj to char* - self.data_raw = data + cdef const uint8_t[::1] data_raw = data cdef unsigned short left cdef unsigned short right @@ -304,11 +304,11 @@ cdef class SMPLayer: cdef size_t i cdef int cmd_offset - self.row_count = self.info.size[1] - self.pcolor.reserve(self.row_count) + cdef size_t row_count = self.info.size[1] + self.pcolor.reserve(row_count) # process bondary table - for i in range(self.row_count): + for i in range(row_count): outline_entry_position = (self.info.outline_table_offset + i * SMPLayer.smp_frame_row_edge.size) @@ -323,7 +323,7 @@ cdef class SMPLayer: self.boundaries.push_back(boundary_def(left, right, False)) # process cmd table - for i in range(self.row_count): + for i in range(row_count): cmd_table_position = (self.info.qdl_table_offset + i * SMPLayer.smp_command_offset.size) @@ -331,257 +331,253 @@ cdef class SMPLayer: data, cmd_table_position)[0] + self.info.frame_offset self.cmd_offsets.push_back(cmd_offset) - def get_picture_data(self, palette): - """ - Convert the palette index matrix to a colored image. - """ - return determine_rgba_matrix(self.pcolor, palette) + for i in range(row_count): + self.pcolor.push_back(self.create_color_row(variant, data_raw, i)) - def get_hotspot(self): + cdef vector[pixel] create_color_row(self, + SMPLayerVariant variant, + const uint8_t[::1] &data_raw, + Py_ssize_t rowid): """ - Return the layer's hotspot (the "center" of the image) + extract colors (pixels) for the given rowid. """ - return self.info.hotspot - def get_palette_number(self): - """ - Return the layer's palette number. + cdef vector[pixel] row_data + cdef Py_ssize_t i - :return: Palette number of the layer. - :rtype: int - """ - return self.pcolor[0][0].palette & 0b00111111 + first_cmd_offset = self.cmd_offsets[rowid] + cdef boundary_def bounds = self.boundaries[rowid] + cdef size_t pixel_count = self.info.size[0] - def __repr__(self): - return repr(self.info) + # preallocate memory + row_data.reserve(pixel_count) + # row is completely transparent + if bounds.full_row: + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) -cdef class SMPMainLayer(SMPLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) + return row_data - for i in range(self.row_count): - self.pcolor.push_back(create_color_row(self, i)) + # start drawing the left transparent space + for i in range(bounds.left): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - def get_damage_mask(self): - """ - Convert the 4th pixel byte to a mask used for damaged units. - """ - return determine_damage_matrix(self.pcolor) + # process the drawing commands for this row. + self.process_drawing_cmds(variant, + data_raw, + row_data, rowid, + first_cmd_offset, + pixel_count - bounds.right) -cdef class SMPShadowLayer(SMPLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) + # finish by filling up the right transparent space + for i in range(bounds.right): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - for i in range(self.row_count): - self.pcolor.push_back(create_color_row(self, i)) + # verify size of generated row + if row_data.size() != pixel_count: + got = row_data.size() + summary = ( + f"{got:d}/{pixel_count:d} -> row {rowid:d}, " + f"layer type {self.info.layer_type:x}, " + f"offset {first_cmd_offset:d} / {first_cmd_offset:#x}" + ) + message = ( + f"got {'LESS' if got < pixel_count else 'MORE'} pixels than expected: {summary}, " + f"missing: {abs(pixel_count - got):d}" + ) -cdef class SMPOutlineLayer(SMPLayer): - def __init__(self, layer_header, data): - super().__init__(layer_header, data) + raise Exception(message) - for i in range(self.row_count): - self.pcolor.push_back(create_color_row(self, i)) + return row_data -ctypedef fused SMPLayerVariant: - SMPMainLayer - SMPShadowLayer - SMPOutlineLayer + @cython.boundscheck(False) + cdef void process_drawing_cmds(self, + SMPLayerVariant variant, + const uint8_t[::1] &data_raw, + vector[pixel] &row_data, + Py_ssize_t rowid, + Py_ssize_t first_cmd_offset, + size_t expected_size): + """ + extract colors (pixels) for the drawing commands + found for this row in the SMP frame. + """ + # position in the data blob, we start at the first command of this row + cdef Py_ssize_t dpos = first_cmd_offset -@cython.boundscheck(False) -cdef vector[pixel] create_color_row(SMPLayerVariant variant, - Py_ssize_t rowid): - """ - extract colors (pixels) for the given rowid. - """ - cdef vector[pixel] row_data - cdef Py_ssize_t i + # is the end of the current row reached? + cdef bool eor = False - cdef Py_ssize_t first_cmd_offset = variant.cmd_offsets[rowid] - cdef boundary_def bounds = variant.boundaries[rowid] - cdef size_t pixel_count = variant.info.size[0] + cdef uint8_t cmd + cdef uint8_t nextbyte + cdef uint8_t lower_crumb + cdef int pixel_count - # preallocate memory - row_data.reserve(pixel_count) + cdef vector[uint8_t] pixel_data + pixel_data.reserve(4) - # row is completely transparent - if bounds.full_row: - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + # work through commands till end of row. + while not eor: + if row_data.size() > expected_size: + raise Exception( + f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + + f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d}" + ) - return row_data + # fetch drawing instruction + cmd = data_raw[dpos] - # start drawing the left transparent space - for i in range(bounds.left): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # process the drawing commands for this row. - process_drawing_cmds(variant, variant.data_raw, - row_data, rowid, - first_cmd_offset, - pixel_count - bounds.right) - - # finish by filling up the right transparent space - for i in range(bounds.right): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # verify size of generated row - if row_data.size() != pixel_count: - got = row_data.size() - summary = ( - f"{got:d}/{pixel_count:d} -> row {rowid:d}, " - f"layer type {variant.info.layer_type:x}, " - f"offset {first_cmd_offset:d} / {first_cmd_offset:#x}" - ) - message = ( - f"got {'LESS' if got < pixel_count else 'MORE'} pixels than expected: {summary}, " - f"missing: {abs(pixel_count - got):d}" - ) + # Last 2 bits store command type + lower_crumb = 0b00000011 & cmd - raise Exception(message) + # opcode: cmd, rowid: rowid - return row_data + if lower_crumb == 0b00000011: + # eol (end of line) command, this row is finished now. + eor = True + if SMPLayerVariant is SMPShadowLayer and row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) -@cython.boundscheck(False) -cdef void process_drawing_cmds(SMPLayerVariant variant, - const uint8_t[::1] & data_raw, - vector[pixel] & row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - size_t expected_size): + continue - """ - extract colors (pixels) for the drawing commands - found for this row in the SMP frame. - """ - - # position in the data blob, we start at the first command of this row - cdef Py_ssize_t dpos = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd = 0 - cdef uint8_t nextbyte = 0 - cdef uint8_t lower_crumb = 0 - cdef int pixel_count = 0 - - cdef vector[uint8_t] pixel_data - pixel_data.reserve(4) - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + - f"with layer type {variant.info.layer_type:#x}, but we have {row_data.size():d} already!" - ) + elif lower_crumb == 0b00000000: + # skip command + # draw 'count' transparent pixels + # count = (cmd >> 2) + 1 - # fetch drawing instruction - cmd = data_raw[dpos] + pixel_count = (cmd >> 2) + 1 - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - # opcode: cmd, rowid: rowid + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - if lower_crumb == 0b00000011: - # eol (end of line) command, this row is finished now. - eor = True + elif lower_crumb == 0b00000001: + # color_list command + # draw the following 'count' pixels + # pixels are stored as 4 byte palette and meta infos + # count = (cmd >> 2) + 1 - if SMPLayerVariant is SMPShadowLayer and row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) + pixel_count = (cmd >> 2) + 1 - continue + for _ in range(pixel_count): + if SMPLayerVariant is SMPShadowLayer: + dpos += 1 + nextbyte = data_raw[dpos] + row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) + elif SMPLayerVariant is SMPOutlineLayer: + # we don't know the color the game wants + # so we just draw index 0 + row_data.push_back(pixel(color_outline, 0, 0, 0, 0)) + elif SMPLayerVariant is SMPMainLayer: + for _ in range(4): + dpos += 1 + pixel_data.push_back(data_raw[dpos]) + + row_data.push_back(pixel(color_standard, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data.clear() + + elif lower_crumb == 0b00000010 and (SMPLayerVariant is SMPMainLayer or SMPLayerVariant is SMPShadowLayer): + # player_color command + # draw the following 'count' pixels + # pixels are stored as 4 byte palette and meta infos + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + if SMPLayerVariant is SMPShadowLayer: + dpos += 1 + nextbyte = data_raw[dpos] + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + else: + for _ in range(4): + dpos += 1 + pixel_data.push_back(data_raw[dpos]) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data.clear() - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 + else: + raise Exception( + f"unknown smp {self.info.layer_type} layer drawing command: " + + f"{cmd:#x} in row {rowid:d}" + ) - pixel_count = (cmd >> 2) + 1 + # process next command + dpos += 1 - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + # end of row reached, return the created pixel array. + return - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # pixels are stored as 4 byte palette and meta infos - # count = (cmd >> 2) + 1 - pixel_count = (cmd >> 2) + 1 + def get_picture_data(self, palette): + """ + Convert the palette index matrix to a colored image. + """ + return determine_rgba_matrix(self.pcolor, palette) - for _ in range(pixel_count): - if SMPLayerVariant is SMPShadowLayer: - dpos += 1 - nextbyte = data_raw[dpos] - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - elif SMPLayerVariant is SMPOutlineLayer: - # we don't know the color the game wants - # so we just draw index 0 - row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) - elif SMPLayerVariant is SMPMainLayer: - for _ in range(4): - dpos += 1 - pixel_data.push_back(data_raw[dpos]) + def get_hotspot(self): + """ + Return the layer's hotspot (the "center" of the image) + """ + return self.info.hotspot - row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here - pixel_data.clear() + def get_palette_number(self): + """ + Return the layer's palette number. - elif lower_crumb == 0b00000010 and (SMPLayerVariant is SMPMainLayer or SMPLayerVariant is SMPShadowLayer): - # player_color command - # draw the following 'count' pixels - # pixels are stored as 4 byte palette and meta infos - # count = (cmd >> 2) + 1 + :return: Palette number of the layer. + :rtype: int + """ + return self.pcolor[0][0].palette & 0b00111111 - pixel_count = (cmd >> 2) + 1 + def __repr__(self): + return repr(self.info) - for _ in range(pixel_count): - if SMPLayerVariant is SMPShadowLayer: - dpos += 1 - nextbyte = data_raw[dpos] - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - else: - for _ in range(4): - dpos += 1 - pixel_data.push_back(data_raw[dpos]) - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here - pixel_data.clear() +cdef class SMPMainLayer(SMPLayer): + """ + SMPLayer for the main graphics sprite. + """ - else: - raise Exception( - f"unknown smp {variant.info.layer_type} layer drawing command: " + - f"{cmd:#x} in row {rowid:d}" - ) + def __init__(self, layer_header, data): + self.init(self ,layer_header, data) - # process next command - dpos += 1 + def get_damage_mask(self): + """ + Convert the 4th pixel byte to a mask used for damaged units. + """ + return determine_damage_matrix(self.pcolor) - # end of row reached, return the created pixel array. - return +cdef class SMPShadowLayer(SMPLayer): + """ + SMPLayer for the shadow graphics. + """ + def __init__(self, layer_header, data): + self.init(self ,layer_header, data) +cdef class SMPOutlineLayer(SMPLayer): + def __init__(self, layer_header, data): + self.init(self, layer_header, data) @cython.boundscheck(False) @@ -690,7 +686,6 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, return array_data - @cython.boundscheck(False) @cython.wraparound(False) cdef numpy.ndarray determine_damage_matrix(vector[vector[pixel]] &image_matrix): diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index 52e17428c8..2e60a84a00 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -68,357 +68,6 @@ ctypedef fused SMXLayerVariant: SMXShadowLayer -cdef inline(int, int, int, vector[pixel]) create_color_row(SMXLayerVariant variant, - Py_ssize_t rowid): - """ - Extract colors (pixels) for the given rowid. - - :param rowid: Index of the current row in the layer. - :param cmd_offset: Offset of the command table of the layer. - :param color_offset: Offset of the color table of the layer. - :param chunk_pos: Current position in the compressed chunk. - """ - - cdef vector[pixel] row_data - cdef Py_ssize_t i - - cdef int first_cmd_offset = variant.cmd_offset - cdef int first_color_offset = variant.color_offset - cdef int first_chunk_pos = variant.chunk_pos - cdef boundary_def bounds = variant.boundaries[rowid] - cdef size_t pixel_count = variant.info.size[0] - - # preallocate memory - row_data.reserve(pixel_count) - - # row is completely transparent - if bounds.full_row: - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - return variant.cmd_offset, variant.color_offset, variant.chunk_pos, row_data - - # start drawing the left transparent space - for i in range(bounds.left): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # process the drawing commands for this row. - next_cmd_offset, next_color_offset, next_chunk_pos, row_data = \ - process_drawing_cmds(variant, - variant.data_raw, - row_data, - rowid, - first_cmd_offset, - first_color_offset, - first_chunk_pos, - pixel_count - bounds.right - ) - - # finish by filling up the right transparent space - for i in range(bounds.right): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - # verify size of generated row - if row_data.size() != pixel_count: - got = row_data.size() - summary = "%d/%d -> row %d, layer type %s, offset %d / %#x" % ( - got, pixel_count, rowid, variant.info.layer_type, - first_cmd_offset, first_cmd_offset - ) - txt = "got %%s pixels than expected: %s, missing: %d" % ( - summary, abs(pixel_count - got)) - - raise Exception(txt % ("LESS" if got < pixel_count else "MORE")) - - return next_cmd_offset, next_color_offset, next_chunk_pos, row_data - - -@cython.boundscheck(False) -cdef inline(int, int, int, vector[pixel]) process_drawing_cmds(SMXLayerVariant variant, - const uint8_t[::1] & data_raw, - vector[pixel] & row_data, - Py_ssize_t rowid, - Py_ssize_t first_cmd_offset, - Py_ssize_t first_color_offset, - int chunk_pos, - size_t expected_size): - """ - TODO: docstring - """ - # position in the command array, we start at the first command of this row - cdef Py_ssize_t dpos_cmd = first_cmd_offset - - # is the end of the current row reached? - cdef bool eor = False - - cdef uint8_t cmd = 0 - cdef uint8_t lower_crumb = 0 - cdef int pixel_count = 0 - cdef Py_ssize_t dpos_color = 0 - cdef vector[uint8_t] pixel_data - # Pixel data temporary values that need further decompression - cdef uint8_t pixel_data_odd_0 = 0 - cdef uint8_t pixel_data_odd_1 = 0 - cdef uint8_t pixel_data_odd_2 = 0 - cdef uint8_t pixel_data_odd_3 = 0 - # Mask for even indices - # cdef uint8_t pixel_mask_even_0 = 0xFF - cdef uint8_t pixel_mask_even_1 = 0b00000011 - cdef uint8_t pixel_mask_even_2 = 0b11110000 - cdef uint8_t pixel_mask_even_3 = 0b00111111 - - if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: - # Position in the pixel data array - dpos_color = first_color_offset - - cdef uint8_t palette_section_block = 0 - cdef uint8_t palette_section = 0 - cdef uint8_t nextbyte = 0 - - # work through commands till end of row. - while not eor: - if row_data.size() > expected_size: - raise Exception( - f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " - f"with layer type {variant:#x}, but we have {row_data.size():d} " - f"already!" - ) - - # fetch drawing instruction - cmd = data_raw[dpos_cmd] - - # Last 2 bits store command type - lower_crumb = 0b00000011 & cmd - if lower_crumb == 0b00000011: - # eor (end of row) command, this row is finished now. - - eor = True - dpos_cmd += 1 - - # shadows sometimes need an extra pixel at - # the end - if SMXLayerVariant is SMXShadowLayer: - if row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - continue - - elif lower_crumb == 0b00000000: - # skip command - # draw 'count' transparent pixels - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) - - elif lower_crumb == 0b00000001: - # color_list command - # draw the following 'count' pixels - # pixels are stored in 5 byte chunks - # even pixel indices have their info stored - # in byte[0] - byte[3]. odd pixel indices have - # their info stored in byte[1] - byte[4]. - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - if SMXLayerVariant is SMXMainLayer8to5: - pixel_data.reserve(4) - for _ in range(pixel_count): - # Start fetching pixel data - if chunk_pos: - # Odd indices require manual extraction of each of the 4 values - - # Palette index. Essentially a rotation of (byte[1]byte[2]) - # by 6 to the left, then masking with 0x00FF. - pixel_data_odd_0 = data_raw[dpos_color + 1] - pixel_data_odd_1 = data_raw[dpos_color + 2] - pixel_data.push_back( - (pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) - - # Palette section. Described in byte[2] in bits 4-5. - pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) - - # Damage mask 1. Essentially a rotation of (byte[3]byte[4]) - # by 6 to the left, then masking with 0x00F0. - pixel_data_odd_2 = data_raw[dpos_color + 3] - pixel_data_odd_3 = data_raw[dpos_color + 4] - pixel_data.push_back( - ((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) - - # Damage mask 2. Described in byte[4] in bits 0-5. - pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) - - row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3])) - - # Go to next pixel - dpos_color += 5 - - else: - # Even indices can be read "as is". They just have to be masked. - for i in range(4): - pixel_data.push_back(data_raw[dpos_color + i]) - - row_data.push_back(pixel(color_standard, - pixel_data[0], - pixel_data[1] & pixel_mask_even_1, - pixel_data[2] & pixel_mask_even_2, - pixel_data[3] & pixel_mask_even_3)) - - odd = not odd - pixel_data.clear() - - if SMXLayerVariant is SMXMainLayer4plus1: - palette_section_block = data_raw[dpos_color + (4 - chunk_pos)] - - for _ in range(pixel_count): - # Start fetching pixel data - palette_section = ( - palette_section_block >> (2 * chunk_pos)) & 0x03 - row_data.push_back(pixel(color_standard, - data_raw[dpos_color], - palette_section, - 0, - 0)) - - dpos_color += 1 - chunk_pos += 1 - - # Skip to next chunk - if chunk_pos > 3: - chunk_pos = 0 - dpos_color += 1 # Skip palette section block - palette_section_block = data_raw[dpos_color + 4] - - if SMXLayerVariant is SMXShadowLayer: - for _ in range(pixel_count): - dpos_cmd += 1 - nextbyte = data_raw[dpos_cmd] - - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - - if SMXLayerVariant is SMXOutlineLayer: - # we don't know the color the game wants - # so we just draw index 0 - row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) - - elif lower_crumb == 0b00000010: - if SMXLayerVariant is SMXMainLayer8to5: - # player_color command - # draw the following 'count' pixels - # pixels are stored in 5 byte chunks - # even pixel indices have their info stored - # in byte[0] - byte[3]. odd pixel indices have - # their info stored in byte[1] - byte[4]. - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # Start fetching pixel data - if odd: - # Odd indices require manual extraction of each of the 4 values - - # Palette index. Essentially a rotation of (byte[1]byte[2]) - # by 6 to the left, then masking with 0x00FF. - pixel_data_odd_0 = data_raw[dpos_color + 1] - pixel_data_odd_1 = data_raw[dpos_color + 2] - pixel_data.push_back( - (pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) - - # Palette section. Described in byte[2] in bits 4-5. - pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) - - # Damage modifier 1. Essentially a rotation of (byte[3]byte[4]) - # by 6 to the left, then masking with 0x00F0. - pixel_data_odd_2 = data_raw[dpos_color + 3] - pixel_data_odd_3 = data_raw[dpos_color + 4] - pixel_data.push_back( - ((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) - - # Damage modifier 2. Described in byte[4] in bits 0-5. - pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) - - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3])) - - # Go to next pixel - dpos_color += 5 - - else: - # Even indices can be read "as is". They just have to be masked. - for px_dpos in range(4): - pixel_data.push_back( - data_raw[dpos_color + px_dpos]) - - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1] & pixel_mask_even_1, - pixel_data[2] & pixel_mask_even_2, - pixel_data[3] & pixel_mask_even_3)) - - odd = not odd - pixel_data.clear() - - elif SMXLayerVariant is SMXMainLayer4plus1: - # player_color command - # draw the following 'count' pixels - # 4 pixels are stored in every 5 byte chunk. - # palette indices are contained in byte[0] - byte[3] - # palette sections are stored in byte[4] - # count = (cmd >> 2) + 1 - - pixel_count = (cmd >> 2) + 1 - - for _ in range(pixel_count): - # Start fetching pixel data - palette_section = ( - palette_section_block >> (2 * chunk_pos)) & 0x03 - row_data.push_back(pixel(color_player, - data_raw[dpos_color], - palette_section, - 0, - 0)) - - dpos_color += 1 - chunk_pos += 1 - - # Skip to next chunk - if chunk_pos > 3: - chunk_pos = 0 - dpos_color += 1 # Skip palette section block - palette_section_block = data_raw[dpos_color + 4] - - else: - raise Exception( - f"unknown smx main graphics layer drawing command: " + - f"{cmd:#x} in row {rowid:d}" - ) - - # Process next command - dpos_cmd += 1 - - if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: - return dpos_cmd, dpos_color, chunk_pos, row_data - if SMXLayerVariant is SMXOutlineLayer or SMXLayerVariant is SMXShadowLayer: - return dpos_cmd, dpos_cmd, chunk_pos, row_data - class SMX: """ @@ -678,22 +327,7 @@ cdef class SMXLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor - # memory pointer - cdef const uint8_t[::1] data_raw - - # current command - cdef int cmd_offset - - # current color - cdef int color_offset - - # current chunk position - cdef int chunk_pos - - # rows - cdef size_t row_count - - def __init__(self, layer_header, data): + def init(self, SMXLayerVariant variant, layer_header, data): """ SMX layer definition superclass. There can be various types of layers inside an SMX frame. @@ -708,18 +342,19 @@ cdef class SMXLayer: if not (isinstance(data, bytes) or isinstance(data, bytearray)): raise ValueError("Layer data must be some bytes object") + # memory pointer # convert the bytes obj to char* - self.data_raw = data + cdef const uint8_t[::1] data_raw = data cdef unsigned short left cdef unsigned short right cdef size_t i - self.row_count = self.info.size[1] - self.pcolor.reserve(self.row_count) + cdef size_t row_count = self.info.size[1] + self.pcolor.reserve(row_count) # process bondary table - for i in range(self.row_count): + for i in range(row_count): outline_entry_position = (self.info.outline_table_offset + i * SMXLayer.smp_layer_row_edge.size) @@ -733,9 +368,379 @@ cdef class SMXLayer: else: self.boundaries.push_back(boundary_def(left, right, False)) - self.cmd_offset = self.info.qdl_command_table_offset - self.color_offset = self.info.qdl_color_table_offset - self.chunk_pos = 0 + cdef int cmd_offset = self.info.qdl_command_table_offset + cdef int color_offset = self.info.qdl_color_table_offset + cdef int chunk_pos = 0 + + # process cmd table + for i in range(row_count): + cmd_offset, color_offset, chunk_pos, row_data = \ + self.create_color_row(variant, data_raw, i, cmd_offset, color_offset, chunk_pos) + + self.pcolor.push_back(row_data) + + + cdef inline (int, int, int, vector[pixel]) create_color_row(self, + SMXLayerVariant variant, + const uint8_t[::1] &data_raw, + Py_ssize_t rowid, + int cmd_offset, + int color_offset, + int chunk_pos): + """ + Extract colors (pixels) for the given rowid. + + :param rowid: Index of the current row in the layer. + :param cmd_offset: Offset of the command table of the layer. + :param color_offset: Offset of the color table of the layer. + :param chunk_pos: Current position in the compressed chunk. + """ + + cdef vector[pixel] row_data + cdef Py_ssize_t i + + cdef int first_cmd_offset = cmd_offset + cdef int first_color_offset = color_offset + cdef boundary_def bounds = self.boundaries[rowid] + cdef size_t pixel_count = self.info.size[0] + + # preallocate memory + row_data.reserve(pixel_count) + + # row is completely transparent + if bounds.full_row: + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + return cmd_offset, color_offset, chunk_pos, row_data + + # start drawing the left transparent space + for i in range(bounds.left): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + # process the drawing commands for this row. + next_cmd_offset, next_color_offset, chunk_pos, row_data = \ + self.process_drawing_cmds( + variant, + data_raw, + row_data, + rowid, + first_cmd_offset, + first_color_offset, + chunk_pos, + pixel_count - bounds.right + ) + + # finish by filling up the right transparent space + for i in range(bounds.right): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + # verify size of generated row + if row_data.size() != pixel_count: + got = row_data.size() + summary = "%d/%d -> row %d, layer type %d, offset %d / %#x" % ( + got, pixel_count, rowid, self.info.layer_type, + first_cmd_offset, first_cmd_offset + ) + txt = "got %%s pixels than expected: %s, missing: %d" % ( + summary, abs(pixel_count - got)) + + raise Exception(txt % ("LESS" if got < pixel_count else "MORE")) + + return next_cmd_offset, next_color_offset, chunk_pos, row_data + + + @cython.boundscheck(False) + cdef inline (int, int, int, vector[pixel]) process_drawing_cmds(self, + SMXLayerVariant variant, + const uint8_t[::1] &data_raw, + vector[pixel] &row_data, + Py_ssize_t rowid, + Py_ssize_t first_cmd_offset, + Py_ssize_t first_color_offset, + int chunk_pos, + size_t expected_size): + """ + extract colors (pixels) for the drawing commands that were + compressed with 8to5 compression. + """ + # position in the command array, we start at the first command of this row + cdef Py_ssize_t dpos_cmd = first_cmd_offset + + # Position in the pixel data array + cdef Py_ssize_t dpos_color = first_color_offset + + # Position in the compression chunk. + cdef bool odd = chunk_pos + cdef int px_dpos = 0 # For loop iterator + + # is the end of the current row reached? + cdef bool eor = False + + cdef uint8_t cmd = 0 + cdef uint8_t lower_crumb = 0 + cdef int pixel_count = 0 + cdef vector[uint8_t] pixel_data + pixel_data.reserve(4) + + # Pixel data temporary values that need further decompression + cdef uint8_t pixel_data_odd_0 = 0 + cdef uint8_t pixel_data_odd_1 = 0 + cdef uint8_t pixel_data_odd_2 = 0 + cdef uint8_t pixel_data_odd_3 = 0 + + # Mask for even indices + # cdef uint8_t pixel_mask_even_0 = 0xFF + cdef uint8_t pixel_mask_even_1 = 0b00000011 + cdef uint8_t pixel_mask_even_2 = 0b11110000 + cdef uint8_t pixel_mask_even_3 = 0b00111111 + + if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: + # Position in the pixel data array + dpos_color = first_color_offset + + cdef uint8_t palette_section_block = 0 + cdef uint8_t palette_section = 0 + cdef uint8_t nextbyte = 0 + + # work through commands till end of row. + while not eor: + if row_data.size() > expected_size: + raise Exception( + f"Only {expected_size:d} pixels should be drawn in row {rowid:d} " + f"with layer type {self.info.layer_type:#x}, but we have {row_data.size():d} " + f"already!" + ) + + # fetch drawing instruction + cmd = data_raw[dpos_cmd] + + # Last 2 bits store command type + lower_crumb = 0b00000011 & cmd + + if lower_crumb == 0b00000011: + # eor (end of row) command, this row is finished now. + eor = True + dpos_cmd += 1 + + # shadows sometimes need an extra pixel at + # the end + if SMXLayerVariant is SMXShadowLayer: + if row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + continue + + elif lower_crumb == 0b00000000: + # skip command + # draw 'count' transparent pixels + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + row_data.push_back(pixel(color_transparent, 0, 0, 0, 0)) + + elif lower_crumb == 0b00000001: + # color_list command + # draw the following 'count' pixels + # pixels are stored in 5 byte chunks + # even pixel indices have their info stored + # in byte[0] - byte[3]. odd pixel indices have + # their info stored in byte[1] - byte[4]. + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + if SMXLayerVariant is SMXMainLayer8to5: + pixel_data.reserve(4) + for _ in range(pixel_count): + # Start fetching pixel data + if odd: + # Odd indices require manual extraction of each of the 4 values + + # Palette index. Essentially a rotation of (byte[1]byte[2]) + # by 6 to the left, then masking with 0x00FF. + pixel_data_odd_0 = data_raw[dpos_color + 1] + pixel_data_odd_1 = data_raw[dpos_color + 2] + pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) + + # Palette section. Described in byte[2] in bits 4-5. + pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) + + # Damage mask 1. Essentially a rotation of (byte[3]byte[4]) + # by 6 to the left, then masking with 0x00F0. + pixel_data_odd_2 = data_raw[dpos_color + 3] + pixel_data_odd_3 = data_raw[dpos_color + 4] + pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) + + # Damage mask 2. Described in byte[4] in bits 0-5. + pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) + + row_data.push_back(pixel(color_standard, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3])) + + # Go to next pixel + dpos_color += 5 + + else: + # Even indices can be read "as is". They just have to be masked. + for i in range(4): + pixel_data.push_back(data_raw[dpos_color + i]) + + row_data.push_back(pixel(color_standard, + pixel_data[0], + pixel_data[1] & pixel_mask_even_1, + pixel_data[2] & pixel_mask_even_2, + pixel_data[3] & pixel_mask_even_3)) + + odd = not odd + pixel_data.clear() + + if SMXLayerVariant is SMXMainLayer4plus1: + palette_section_block = data_raw[dpos_color + (4 - chunk_pos)] + + for _ in range(pixel_count): + # Start fetching pixel data + palette_section = ( + palette_section_block >> (2 * chunk_pos)) & 0x03 + row_data.push_back(pixel(color_standard, + data_raw[dpos_color], + palette_section, + 0, + 0)) + + dpos_color += 1 + chunk_pos += 1 + + # Skip to next chunk + if chunk_pos > 3: + chunk_pos = 0 + dpos_color += 1 # Skip palette section block + palette_section_block = data_raw[dpos_color + 4] + + if SMXLayerVariant is SMXShadowLayer: + for _ in range(pixel_count): + dpos_cmd += 1 + nextbyte = data_raw[dpos_cmd] + + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + + if SMXLayerVariant is SMXOutlineLayer: + # we don't know the color the game wants + # so we just draw index 0 + row_data.push_back(pixel(color_outline, + 0, 0, 0, 0)) + + elif lower_crumb == 0b00000010: + if SMXLayerVariant is SMXMainLayer8to5: + # player_color command + # draw the following 'count' pixels + # pixels are stored in 5 byte chunks + # even pixel indices have their info stored + # in byte[0] - byte[3]. odd pixel indices have + # their info stored in byte[1] - byte[4]. + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + # Start fetching pixel data + if odd: + # Odd indices require manual extraction of each of the 4 values + + # Palette index. Essentially a rotation of (byte[1]byte[2]) + # by 6 to the left, then masking with 0x00FF. + pixel_data_odd_0 = data_raw[dpos_color + 1] + pixel_data_odd_1 = data_raw[dpos_color + 2] + pixel_data.push_back((pixel_data_odd_0 >> 2) | (pixel_data_odd_1 << 6)) + + # Palette section. Described in byte[2] in bits 4-5. + pixel_data.push_back((pixel_data_odd_1 >> 2) & 0x03) + + # Damage modifier 1. Essentially a rotation of (byte[3]byte[4]) + # by 6 to the left, then masking with 0x00F0. + pixel_data_odd_2 = data_raw[dpos_color + 3] + pixel_data_odd_3 = data_raw[dpos_color + 4] + pixel_data.push_back(((pixel_data_odd_2 >> 2) | (pixel_data_odd_3 << 6)) & 0xF0) + + # Damage modifier 2. Described in byte[4] in bits 0-5. + pixel_data.push_back((pixel_data_odd_3 >> 2) & 0x3F) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3])) + + # Go to next pixel + dpos_color += 5 + + else: + # Even indices can be read "as is". They just have to be masked. + for px_dpos in range(4): + pixel_data.push_back(data_raw[dpos_color + px_dpos]) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1] & pixel_mask_even_1, + pixel_data[2] & pixel_mask_even_2, + pixel_data[3] & pixel_mask_even_3)) + + odd = not odd + pixel_data.clear() + + elif SMXLayerVariant is SMXMainLayer4plus1: + # player_color command + # draw the following 'count' pixels + # 4 pixels are stored in every 5 byte chunk. + # palette indices are contained in byte[0] - byte[3] + # palette sections are stored in byte[4] + # count = (cmd >> 2) + 1 + + pixel_count = (cmd >> 2) + 1 + + for _ in range(pixel_count): + # Start fetching pixel data + palette_section = (palette_section_block >> (2 * chunk_pos)) & 0x03 + row_data.push_back(pixel(color_player, + data_raw[dpos_color], + palette_section, + 0, + 0)) + + dpos_color += 1 + chunk_pos += 1 + + # Skip to next chunk + if chunk_pos > 3: + chunk_pos = 0 + dpos_color += 1 # Skip palette section block + palette_section_block = data_raw[dpos_color + 4] + + else: + raise Exception( + f"unknown smx main graphics layer drawing command: " + + f"{cmd:#x} in row {rowid:d}" + ) + + # Process next command + dpos_cmd += 1 + + if SMXLayerVariant is SMXMainLayer8to5 or SMXLayerVariant is SMXMainLayer4plus1: + return dpos_cmd, dpos_color, chunk_pos, row_data + elif SMXLayerVariant is SMXOutlineLayer or SMXLayerVariant is SMXShadowLayer: + return dpos_cmd, dpos_cmd, chunk_pos, row_data + def get_picture_data(self, palette): """ @@ -776,55 +781,22 @@ cdef class SMXMainLayer8to5(SMXLayer): """ def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos - - self.pcolor.push_back(row_data) + self.init(self, layer_header, data) cdef class SMXMainLayer4plus1(SMXLayer): def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos + self.init(self, layer_header, data) - self.pcolor.push_back(row_data) cdef class SMXOutlineLayer(SMXLayer): def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos + self.init(self, layer_header, data) - self.pcolor.push_back(row_data) cdef class SMXShadowLayer(SMXLayer): def __init__(self, layer_header, data): - super().__init__(layer_header, data) - # process cmd table - for i in range(self.row_count): - cmd_offset, color_offset, chunk_pos, row_data = create_color_row( - self, i) - self.cmd_offset = cmd_offset - self.color_offset = color_offset - self.chunk_pos = chunk_pos + self.init(self, layer_header, data) - self.pcolor.push_back(row_data) From 64782e68bbc14c99b2b82bed5d370ce031178d6d Mon Sep 17 00:00:00 2001 From: Jeremiah Morgan Date: Sun, 19 Jan 2025 22:06:07 +0000 Subject: [PATCH 12/17] split conditionals --- .../convert/value_object/read/media/smp.pyx | 65 ++++++++++--------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 9ad253c5c3..40787250ec 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -439,13 +439,14 @@ cdef class SMPLayer: # eol (end of line) command, this row is finished now. eor = True - if SMPLayerVariant is SMPShadowLayer and row_data.size() < expected_size: - # copy the last drawn pixel - # (still stored in nextbyte) - # - # TODO: confirm that this is the - # right way to do it - row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) + if SMPLayerVariant is SMPShadowLayer: + if row_data.size() < expected_size: + # copy the last drawn pixel + # (still stored in nextbyte) + # + # TODO: confirm that this is the + # right way to do it + row_data.push_back(pixel(color_shadow, nextbyte, 0, 0, 0)) continue @@ -488,31 +489,37 @@ cdef class SMPLayer: pixel_data[3] & 0x1F)) # remove "usage" bit here pixel_data.clear() - elif lower_crumb == 0b00000010 and (SMPLayerVariant is SMPMainLayer or SMPLayerVariant is SMPShadowLayer): - # player_color command - # draw the following 'count' pixels - # pixels are stored as 4 byte palette and meta infos - # count = (cmd >> 2) + 1 + elif lower_crumb == 0b00000010: + if (SMPLayerVariant is SMPMainLayer or SMPLayerVariant is SMPShadowLayer): + # player_color command + # draw the following 'count' pixels + # pixels are stored as 4 byte palette and meta infos + # count = (cmd >> 2) + 1 - pixel_count = (cmd >> 2) + 1 + pixel_count = (cmd >> 2) + 1 - for _ in range(pixel_count): - if SMPLayerVariant is SMPShadowLayer: - dpos += 1 - nextbyte = data_raw[dpos] - row_data.push_back(pixel(color_shadow, - nextbyte, 0, 0, 0)) - else: - for _ in range(4): + for _ in range(pixel_count): + if SMPLayerVariant is SMPShadowLayer: dpos += 1 - pixel_data.push_back(data_raw[dpos]) - - row_data.push_back(pixel(color_player, - pixel_data[0], - pixel_data[1], - pixel_data[2], - pixel_data[3] & 0x1F)) # remove "usage" bit here - pixel_data.clear() + nextbyte = data_raw[dpos] + row_data.push_back(pixel(color_shadow, + nextbyte, 0, 0, 0)) + else: + for _ in range(4): + dpos += 1 + pixel_data.push_back(data_raw[dpos]) + + row_data.push_back(pixel(color_player, + pixel_data[0], + pixel_data[1], + pixel_data[2], + pixel_data[3] & 0x1F)) # remove "usage" bit here + pixel_data.clear() + else: + raise Exception( + f"unknown smp {self.info.layer_type} layer drawing command: " + + f"{cmd:#x} in row {rowid:d}" + ) else: raise Exception( From 71391d3ff2f63554e9bdf2bc686c859885a5c3b1 Mon Sep 17 00:00:00 2001 From: jere8184 Date: Fri, 24 Jan 2025 21:00:46 +0000 Subject: [PATCH 13/17] Update .mailmap --- .mailmap | 3 --- 1 file changed, 3 deletions(-) diff --git a/.mailmap b/.mailmap index 3d8b8ec12e..5acbd1b581 100644 --- a/.mailmap +++ b/.mailmap @@ -21,8 +21,5 @@ Tobias Feldballe Jonas Borchelt Derek Frogget <114030121+derekfrogget@users.noreply.github.com> Nikhil Ghosh -<<<<<<< HEAD David Wever <56411717+dmwever@users.noreply.github.com> -======= Ngô Xuân Minh ->>>>>>> e8251ff6 (copying.md) From 68e500b65de7416b051cc030c9d437948d32b2e6 Mon Sep 17 00:00:00 2001 From: heinezen Date: Mon, 27 Jan 2025 03:22:14 +0100 Subject: [PATCH 14/17] convert: Make fused type work without inheritance. --- .../convert/value_object/read/media/smx.pyx | 58 ++++++++----------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index 2e60a84a00..7fb6a39f7f 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -61,12 +61,24 @@ cdef public dict LAYER_TYPES = { } +cdef class SMXMainLayer8to5: + pass + +cdef class SMXMainLayer4plus1: + pass + +cdef class SMXShadowLayer: + pass + +cdef class SMXOutlineLayer: + pass + + ctypedef fused SMXLayerVariant: SMXMainLayer8to5 SMXMainLayer4plus1 - SMXOutlineLayer SMXShadowLayer - + SMXOutlineLayer class SMX: @@ -190,16 +202,17 @@ class SMX: if layer_type is SMXLayerType.MAIN: if layer_header.compression_type == 0x08: - self.main_frames.append(SMXMainLayer8to5(layer_header, data)) + self.main_frames.append(SMXLayer(SMXMainLayer8to5(), layer_header, data)) elif layer_header.compression_type == 0x00: - self.main_frames.append(SMXMainLayer4plus1(layer_header, data)) + self.main_frames.append(SMXLayer(SMXMainLayer4plus1(), layer_header, data)) elif layer_type is SMXLayerType.SHADOW: - self.shadow_frames.append(SMXShadowLayer(layer_header, data)) + self.shadow_frames.append(SMXLayer(SMXShadowLayer(), layer_header, data)) elif layer_type is SMXLayerType.OUTLINE: - self.outline_frames.append(SMXOutlineLayer(layer_header, data)) + # self.outline_frames.append(SMXLayer(SMXOutlineLayer(), layer_header, data)) + pass def get_frames(self, layer: int = 0): """ @@ -327,6 +340,9 @@ cdef class SMXLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor + def __init__(self, variant, layer_header, data) -> None: + self.init(variant, layer_header, data) + def init(self, SMXLayerVariant variant, layer_header, data): """ SMX layer definition superclass. There can be various types of @@ -376,7 +392,6 @@ cdef class SMXLayer: for i in range(row_count): cmd_offset, color_offset, chunk_pos, row_data = \ self.create_color_row(variant, data_raw, i, cmd_offset, color_offset, chunk_pos) - self.pcolor.push_back(row_data) @@ -438,8 +453,8 @@ cdef class SMXLayer: # verify size of generated row if row_data.size() != pixel_count: got = row_data.size() - summary = "%d/%d -> row %d, layer type %d, offset %d / %#x" % ( - got, pixel_count, rowid, self.info.layer_type, + summary = "%d/%d -> row %d, layer type %s, offset %d / %#x" % ( + got, pixel_count, rowid, repr(self.info.layer_type), first_cmd_offset, first_cmd_offset ) txt = "got %%s pixels than expected: %s, missing: %d" % ( @@ -775,31 +790,6 @@ cdef class SMXLayer: return repr(self.info) -cdef class SMXMainLayer8to5(SMXLayer): - """ - Compressed SMP layer (compression type 8to5) for the main graphics sprite. - """ - - def __init__(self, layer_header, data): - self.init(self, layer_header, data) - -cdef class SMXMainLayer4plus1(SMXLayer): - def __init__(self, layer_header, data): - self.init(self, layer_header, data) - - -cdef class SMXOutlineLayer(SMXLayer): - def __init__(self, layer_header, data): - self.init(self, layer_header, data) - - -cdef class SMXShadowLayer(SMXLayer): - def __init__(self, layer_header, data): - self.init(self, layer_header, data) - - - - @cython.boundscheck(False) @cython.wraparound(False) cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, From 09cfc3d20b08a442e46d67a11ee956f931ed9143 Mon Sep 17 00:00:00 2001 From: heinezen Date: Mon, 27 Jan 2025 03:47:49 +0100 Subject: [PATCH 15/17] convert: Fix missing loop in outline layer parsing. --- openage/convert/value_object/read/media/smx.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index 7fb6a39f7f..565dcb611e 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -211,8 +211,7 @@ class SMX: self.shadow_frames.append(SMXLayer(SMXShadowLayer(), layer_header, data)) elif layer_type is SMXLayerType.OUTLINE: - # self.outline_frames.append(SMXLayer(SMXOutlineLayer(), layer_header, data)) - pass + self.outline_frames.append(SMXLayer(SMXOutlineLayer(), layer_header, data)) def get_frames(self, layer: int = 0): """ @@ -653,8 +652,9 @@ cdef class SMXLayer: if SMXLayerVariant is SMXOutlineLayer: # we don't know the color the game wants # so we just draw index 0 - row_data.push_back(pixel(color_outline, - 0, 0, 0, 0)) + for _ in range(pixel_count): + row_data.push_back(pixel(color_outline, + 0, 0, 0, 0)) elif lower_crumb == 0b00000010: if SMXLayerVariant is SMXMainLayer8to5: From cb5f61e3b25c269bf54c69e163c2faccf8487f7b Mon Sep 17 00:00:00 2001 From: heinezen Date: Sun, 9 Feb 2025 21:31:49 +0100 Subject: [PATCH 16/17] convert: Make fused type work without inheritance. --- .../convert/value_object/read/media/smp.pyx | 66 +++++++++---------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index 40787250ec..a67efd87bb 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -62,6 +62,24 @@ cdef public dict LAYER_TYPES = { } +cdef class SMPMainLayer: + pass + + +cdef class SMPShadowLayer: + pass + + +cdef class SMPOutlineLayer: + pass + + +ctypedef fused SMPLayerVariant: + SMPMainLayer + SMPShadowLayer + SMPOutlineLayer + + class SMP: """ Class for reading/converting the SMP image format (successor of SLP). @@ -155,16 +173,16 @@ class SMP: if layer_header.layer_type == 0x02: # layer that store the main graphic - self.main_frames.append(SMPMainLayer(layer_header, data)) + self.main_frames.append(SMPLayer(SMPMainLayer(), layer_header, data)) elif layer_header.layer_type == 0x04: # layer that stores a shadow - self.shadow_frames.append(SMPShadowLayer(layer_header, data)) + self.shadow_frames.append(SMPLayer(SMPShadowLayer(), layer_header, data)) elif layer_header.layer_type == 0x08 or \ layer_header.layer_type == 0x10: # layer that stores an outline - self.outline_frames.append(SMPOutlineLayer(layer_header, data)) + self.outline_frames.append(SMPLayer(SMPOutlineLayer(), layer_header, data)) else: raise Exception( @@ -253,11 +271,6 @@ class SMPLayerHeader: ) return "".join(ret) -ctypedef fused SMPLayerVariant: - SMPMainLayer - SMPShadowLayer - SMPOutlineLayer - cdef class SMPLayer: """ @@ -288,6 +301,9 @@ cdef class SMPLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor + def __init__(self, variant, layer_header, data) -> None: + self.init(variant, layer_header, data) + def init(self, SMPLayerVariant variant, layer_header, data): self.info = layer_header @@ -540,6 +556,12 @@ cdef class SMPLayer: """ return determine_rgba_matrix(self.pcolor, palette) + def get_damage_mask(self): + """ + Convert the 4th pixel byte to a mask used for damaged units. + """ + return determine_damage_matrix(self.pcolor) + def get_hotspot(self): """ Return the layer's hotspot (the "center" of the image) @@ -559,34 +581,6 @@ cdef class SMPLayer: return repr(self.info) -cdef class SMPMainLayer(SMPLayer): - """ - SMPLayer for the main graphics sprite. - """ - - def __init__(self, layer_header, data): - self.init(self ,layer_header, data) - - def get_damage_mask(self): - """ - Convert the 4th pixel byte to a mask used for damaged units. - """ - return determine_damage_matrix(self.pcolor) - - -cdef class SMPShadowLayer(SMPLayer): - """ - SMPLayer for the shadow graphics. - """ - - def __init__(self, layer_header, data): - self.init(self ,layer_header, data) - -cdef class SMPOutlineLayer(SMPLayer): - def __init__(self, layer_header, data): - self.init(self, layer_header, data) - - @cython.boundscheck(False) @cython.wraparound(False) cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, From 6f8689d617ce6e08e30c4838db28b6586f5a98ca Mon Sep 17 00:00:00 2001 From: heinezen Date: Sun, 9 Feb 2025 21:56:20 +0100 Subject: [PATCH 17/17] convert: Add comments to SMP/SMX code. --- .../convert/value_object/read/media/smp.pyx | 137 ++++++++++++++---- .../convert/value_object/read/media/smx.pyx | 111 ++++++++------ 2 files changed, 180 insertions(+), 68 deletions(-) diff --git a/openage/convert/value_object/read/media/smp.pyx b/openage/convert/value_object/read/media/smp.pyx index a67efd87bb..03de7dd200 100644 --- a/openage/convert/value_object/read/media/smp.pyx +++ b/openage/convert/value_object/read/media/smp.pyx @@ -22,6 +22,7 @@ from libcpp.vector cimport vector endianness = "< " +# Boundary of a row in a SMP layer. cdef struct boundary_def: Py_ssize_t left Py_ssize_t right @@ -63,17 +64,27 @@ cdef public dict LAYER_TYPES = { cdef class SMPMainLayer: + """ + Main graphic layer of an SMP. Stores the color information. + """ pass cdef class SMPShadowLayer: + """ + Shadow layer of an SMP. + """ pass cdef class SMPOutlineLayer: + """ + Outline layer of an SMP. + """ pass +# fused type for the layer variants ctypedef fused SMPLayerVariant: SMPMainLayer SMPShadowLayer @@ -83,7 +94,7 @@ ctypedef fused SMPLayerVariant: class SMP: """ Class for reading/converting the SMP image format (successor of SLP). - This format is used to store all graphics within AoE2: Definitive Edition. + This format is used to store all graphics within AoE2: Definitive Edition (Beta). """ # struct smp_header { @@ -122,15 +133,23 @@ class SMP: # }; smp_layer_header = Struct(endianness + "i i i i I I I I") - def __init__(self, data): + def __init__(self, data: bytes) -> None: + """ + Read the SMP file and store the frames in the object. + + :param data: SMP file data. + """ smp_header = SMP.smp_header.unpack_from(data) signature, version, frame_count, facet_count, frames_per_facet,\ checksum, file_size, source_format, comment = smp_header dbg("SMP") + spam(" signature: %s", signature.decode('ascii')) + spam(" version: %s", version) dbg(" frame count: %s", frame_count) dbg(" facet count: %s", facet_count) dbg(" facets per animation: %s", frames_per_facet) + spam(" checksum: %s", checksum) dbg(" file size: %s B", file_size) dbg(" source format: %s", source_format) dbg(" comment: %s", comment.decode('ascii')) @@ -188,9 +207,10 @@ class SMP: raise Exception( f"unknown layer type: {layer_header.layer_type:#x} at offset {layer_header_offset:#x}" ) + spam(layer_header) - def get_frames(self, layer: int = 0): + def get_frames(self, layer: int = 0) -> list[SMPLayer]: """ Get the frames in the SMP. @@ -198,7 +218,6 @@ class SMP: - 0 = main graphics - 1 = shadow graphics - 2 = outline - :type layer: int """ cdef list frames @@ -221,7 +240,7 @@ class SMP: return frames - def __str__(self): + def __str__(self) -> str: ret = list() ret.extend([repr(self), "\n", SMPLayerHeader.repr_header(), "\n"]) @@ -229,15 +248,39 @@ class SMP: ret.extend([repr(frame), "\n"]) return "".join(ret) - def __repr__(self): + def __repr__(self) -> str: return f"SMP image<{len(self.main_frames):d} frames>" class SMPLayerHeader: - def __init__(self, width, height, hotspot_x, - hotspot_y, layer_type, outline_table_offset, - qdl_table_offset, flags, - frame_offset): + """ + Header of a layer in the SMP file. + """ + def __init__( + self, + width: int, + height: int, + hotspot_x: int, + hotspot_y: int, + layer_type: int, + outline_table_offset: int, + qdl_table_offset: int, + flags: int, + frame_offset: int + ) -> None: + """ + Create a SMP layer header. + + :param width: Width of the layer sprites. + :param height: Height of the layer sprites. + :param hotspot_x: X coordinate of the anchor point. + :param hotspot_y: Y coordinate of the anchor point. + :param layer_type: Type of the layer. + :param outline_table_offset: Offset of the outline table. + :param qdl_table_offset: Offset of the pixel command table. + :param flags: Flags of the layer. + :param frame_offset: Offset of the frame. + """ self.size = (width, height) self.hotspot = (hotspot_x, hotspot_y) @@ -252,16 +295,19 @@ class SMPLayerHeader: # the absolute offset of the frame self.frame_offset = frame_offset + # flags + self.flags = flags + self.palette_number = -1 @staticmethod - def repr_header(): + def repr_header() -> str: return ("width x height | hotspot x/y | " "layer type | " "offset (outline table|qdl table)" ) - def __repr__(self): + def __repr__(self) -> str: ret = ( "% 5d x% 7d | " % self.size, "% 4d /% 5d | " % self.hotspot, @@ -274,7 +320,7 @@ class SMPLayerHeader: cdef class SMPLayer: """ - one layer inside the SMP. you can imagine it as a frame of a video. + Layer inside the SMP. you can imagine it as a frame of an animation. """ # struct smp_frame_row_edge { @@ -301,10 +347,29 @@ cdef class SMPLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor - def __init__(self, variant, layer_header, data) -> None: + def __init__( + self, + variant, # this argument must not be typed because cython can't handle it + layer_header: SMPLayerHeader, + data: bytes + ) -> None: + """ + Create a SMP layer. + + :param variant: Type of the layer. + :param layer_header: Header information of the layer. + :param data: Layer data. + """ self.init(variant, layer_header, data) - def init(self, SMPLayerVariant variant, layer_header, data): + def init(self, SMPLayerVariant variant, layer_header: SMPLayerHeader, data: bytes) -> None: + """ + Read the pixel information of the SMP layer. + + :param variant: Type of the layer. + :param layer_header: Header information of the layer. + :param data: Layer data. + """ self.info = layer_header if not (isinstance(data, bytes) or isinstance(data, bytearray)): @@ -355,7 +420,11 @@ cdef class SMPLayer: const uint8_t[::1] &data_raw, Py_ssize_t rowid): """ - extract colors (pixels) for the given rowid. + Extract colors (pixels) for a pixel row in the layer. + + :param variant: Type of the layer. + :param data_raw: Raw data of the layer. + :param rowid: Index of the current row in the layer. """ cdef vector[pixel] row_data @@ -417,8 +486,14 @@ cdef class SMPLayer: Py_ssize_t first_cmd_offset, size_t expected_size): """ - extract colors (pixels) for the drawing commands - found for this row in the SMP frame. + Extract colors (pixels) from the drawing commands for a row in the layer. + + :param variant: Type of the layer. + :param data_raw: Raw data of the layer. + :param row_data: Stores the extracted pixels. May be prefilled with transparent pixels. + :param rowid: Index of the current row in the layer. + :param first_cmd_offset: Offset of the first drawing command in the data. + :param expected_size: Expected number of pixels in the row. """ # position in the data blob, we start at the first command of this row @@ -550,34 +625,41 @@ cdef class SMPLayer: return - def get_picture_data(self, palette): + def get_picture_data(self, palette) -> numpy.ndarray: """ - Convert the palette index matrix to a colored image. + Convert the palette index matrix to a RGBA image. + + :param palette: Color palette used for pixels in the sprite. + :type palette: .colortable.ColorTable + :return: Array of RGBA values. """ return determine_rgba_matrix(self.pcolor, palette) - def get_damage_mask(self): + def get_damage_mask(self) -> numpy.ndarray: """ Convert the 4th pixel byte to a mask used for damaged units. + + :return: Damage mask of the layer. """ return determine_damage_matrix(self.pcolor) - def get_hotspot(self): + def get_hotspot(self) -> tuple[int, int]: """ - Return the layer's hotspot (the "center" of the image) + Return the layer's hotspot (the "center" of the image). + + :return: Hotspot of the layer. """ return self.info.hotspot - def get_palette_number(self): + def get_palette_number(self) -> int: """ Return the layer's palette number. :return: Palette number of the layer. - :rtype: int """ return self.pcolor[0][0].palette & 0b00111111 - def __repr__(self): + def __repr__(self) -> str: return repr(self.info) @@ -587,6 +669,9 @@ cdef numpy.ndarray determine_rgba_matrix(vector[vector[pixel]] &image_matrix, numpy.ndarray[numpy.uint8_t, ndim=2, mode="c"] palette): """ converts a palette index image matrix to an rgba matrix. + + :param image_matrix: A 2-dimensional array of SMP pixels. + :param palette: Color palette used for normal pixels in the sprite. """ cdef size_t height = image_matrix.size() diff --git a/openage/convert/value_object/read/media/smx.pyx b/openage/convert/value_object/read/media/smx.pyx index 565dcb611e..887440cf7b 100644 --- a/openage/convert/value_object/read/media/smx.pyx +++ b/openage/convert/value_object/read/media/smx.pyx @@ -21,6 +21,8 @@ from libcpp.vector cimport vector # SMX files have little endian byte order endianness = "< " + +# Boundary of a row in a SMX layer. cdef struct boundary_def: Py_ssize_t left Py_ssize_t right @@ -62,18 +64,31 @@ cdef public dict LAYER_TYPES = { cdef class SMXMainLayer8to5: + """ + Main graphics layer of an SMX (compressed with 8to5). + """ pass cdef class SMXMainLayer4plus1: + """ + Main graphics layer of an SMX (compressed with 4plus1). + """ pass cdef class SMXShadowLayer: + """ + Shadow layer of an SMX. + """ pass cdef class SMXOutlineLayer: + """ + Outline layer of an SMX. + """ pass +# fused type for SMX layer variants ctypedef fused SMXLayerVariant: SMXMainLayer8to5 SMXMainLayer4plus1 @@ -114,14 +129,12 @@ class SMX: # }; smx_layer_header = Struct(endianness + "H H h h I i") - def __init__(self, data): + def __init__(self, data: bytes): """ - Read an SMX image file. + Read an SMX image file and store the frames in the object. - :param data: File content as bytes. - :type data: bytes, bytearray + :param data: SMX file data. """ - smx_header = SMX.smx_header.unpack_from(data) self.smp_type, version, frame_count, file_size_comp,\ file_size_uncomp, comment = smx_header @@ -213,7 +226,7 @@ class SMX: elif layer_type is SMXLayerType.OUTLINE: self.outline_frames.append(SMXLayer(SMXOutlineLayer(), layer_header, data)) - def get_frames(self, layer: int = 0): + def get_frames(self, layer: int = 0) -> list[SMXLayer]: """ Get the frames in the SMX. @@ -221,7 +234,6 @@ class SMX: - 0 = main graphics - 1 = shadow graphics - 2 = outline - :type layer: int """ cdef list frames @@ -257,16 +269,23 @@ class SMX: class SMXLayerHeader: - def __init__(self, layer_type, frame_type, - palette_number, - width, height, hotspot_x, hotspot_y, - outline_table_offset, - qdl_command_table_offset, - qdl_color_table_offset): + def __init__( + self, + layer_type: SMXLayerType, + frame_type: int, + palette_number: int, + width: int, + height: int, + hotspot_x: int, + hotspot_y: int, + outline_table_offset: int, + qdl_command_table_offset: int, + qdl_color_table_offset: int + ) -> None: """ Stores the header of a layer including additional info about its frame. - :param layer_type: Type of layer. Either main. shadow or outline. + :param layer_type: Type of layer. :param frame_type: Type of the frame the layer belongs to. :param palette_number: Palette number used for pixels in the frame. :param width: Width of layer in pixels. @@ -276,16 +295,6 @@ class SMXLayerHeader: :param outline_table_offset: Absolute position of the layer's outline table in the file. :param qdl_command_table_offset: Absolute position of the layer's command table in the file. :param qdl_color_table_offset: Absolute position of the layer's pixel data table in the file. - :type layer_type: str - :type frame_type: int - :type palette_number: int - :type width: int - :type height: int - :type hotspot_x: int - :type hotspot_y: int - :type outline_table_offset: int - :type qdl_command_table_offset: int - :type qdl_color_table_offset: int """ self.size = (width, height) @@ -301,12 +310,12 @@ class SMXLayerHeader: self.qdl_color_table_offset = qdl_color_table_offset @staticmethod - def repr_header(): + def repr_header() -> str: return ("layer type | width x height | " "hotspot x/y | " ) - def __repr__(self): + def __repr__(self) -> str: ret = ( "% s | " % self.layer_type, "% 5d x% 7d | " % self.size, @@ -320,7 +329,7 @@ class SMXLayerHeader: cdef class SMXLayer: """ - one layer inside the compressed SMP. + Layer inside the compressed SMX. """ # struct smp_layer_row_edge { @@ -339,18 +348,28 @@ cdef class SMXLayer: # pixel matrix representing the final image cdef vector[vector[pixel]] pcolor - def __init__(self, variant, layer_header, data) -> None: + def __init__( + self, + variant, # this argument must not be typed because cython can't handle it + layer_header: SMXLayerHeader, + data: bytes + ) -> None: + """ + Create a SMX layer. + + :param variant: Type of the layer. + :param layer_header: Header information of the layer. + :param data: Layer data. + """ self.init(variant, layer_header, data) - def init(self, SMXLayerVariant variant, layer_header, data): + def init(self, SMXLayerVariant variant, layer_header: SMXLayerHeader, data: bytes) -> None: """ - SMX layer definition superclass. There can be various types of - layers inside an SMX frame. + SMX layer definition. There can be various types of layers inside an SMX frame. + :param variant: Type of the layer. :param layer_header: Header definition of the layer. :param data: File content as bytes. - :type layer_header: SMXLayerHeader - :type data: bytes, bytearray """ self.info = layer_header @@ -402,8 +421,10 @@ cdef class SMXLayer: int color_offset, int chunk_pos): """ - Extract colors (pixels) for the given rowid. + Extract colors (pixels) for a pixel row in the layer. + :param variant: Type of the layer. + :param data_raw: Raw data of the layer. :param rowid: Index of the current row in the layer. :param cmd_offset: Offset of the command table of the layer. :param color_offset: Offset of the color table of the layer. @@ -477,6 +498,15 @@ cdef class SMXLayer: """ extract colors (pixels) for the drawing commands that were compressed with 8to5 compression. + + :param variant: Type of the layer. + :param data_raw: Raw data of the layer. + :param row_data: Stores the extracted pixels. May be prefilled with transparent pixels. + :param rowid: Row index. + :param first_cmd_offset: Offset of the first drawing command in the data. + :param first_color_offset: Offset of the first color command in the data. + :param chunk_pos: Current position in the compressed chunk. + :param expected_size: Expected number of pixels in the row. """ # position in the command array, we start at the first command of this row cdef Py_ssize_t dpos_cmd = first_cmd_offset @@ -757,32 +787,29 @@ cdef class SMXLayer: return dpos_cmd, dpos_cmd, chunk_pos, row_data - def get_picture_data(self, palette): + def get_picture_data(self, palette) -> numpy.ndarray: """ Convert the palette index matrix to a RGBA image. - :param main_palette: Color palette used for pixels in the sprite. - :type main_palette: .colortable.ColorTable + :param palette: Color palette used for pixels in the sprite. + :type palette: .colortable.ColorTable :return: Array of RGBA values. - :rtype: numpy.ndarray """ return determine_rgba_matrix(self.pcolor, palette) - def get_hotspot(self): + def get_hotspot(self) -> tuple[int, int]: """ Return the layer's hotspot (the "center" of the image). :return: Hotspot of the layer. - :rtype: tuple """ return self.info.hotspot - def get_palette_number(self): + def get_palette_number(self) -> int: """ Return the layer's palette number. :return: Palette number of the layer. - :rtype: int """ return self.info.palette_number