Store tag markup for serialization

ext/liquid_c/block.c

@@ -8,6 +8,7 @@
 #include "context.h"
 #include "parse_context.h"
 #include "vm_assembler.h"
+#include "tag_markup.h"
 #include <stdio.h>
 
 static ID
@@ -22,15 +23,11 @@ static ID
 static VALUE tag_registry;
 static VALUE variable_placeholder = Qnil;
 
-typedef struct tag_markup {
-    VALUE name;
-    VALUE markup;
-} tag_markup_t;
-
 typedef struct parse_context {
     tokenizer_t *tokenizer;
     VALUE tokenizer_obj;
     VALUE ruby_obj;
+    VALUE parent_tag;
 } parse_context_t;
 
 static void ensure_body_compiled(const block_body_t *body)
@@ -43,6 +40,7 @@ static void ensure_body_compiled(const block_body_t *body)
 static void block_body_mark(void *ptr)
 {
     block_body_t *body = ptr;
+    c_buffer_rb_gc_mark(&body->tags);
     if (body->compiled) {
         document_body_entry_mark(&body->as.compiled.document_body_entry);
         rb_gc_mark(body->as.compiled.nodelist);
@@ -91,8 +89,10 @@ static VALUE block_body_allocate(VALUE klass)
 
     body->compiled = false;
     body->obj = obj;
+    body->tags = c_buffer_init();
     body->as.intermediate.blank = true;
     body->as.intermediate.root = false;
+    body->as.intermediate.bound_to_tag = false;
     body->as.intermediate.render_score = 0;
     body->as.intermediate.vm_assembler_pool = NULL;
     body->as.intermediate.code = NULL;
@@ -125,11 +125,25 @@ static int is_id(int c)
     return rb_isalnum(c) || c == '_';
 }
 
-static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_t *parse_context)
+static void block_body_add_node(block_body_t *body, VALUE node)
+{
+    assert(!body->compiled);
+    c_buffer_write_ruby_value(&body->tags, node);
+    vm_assembler_add_write_node(body->as.intermediate.code);
+}
+
+static void block_body_push_tag_markup(block_body_t *body, VALUE parse_context, VALUE tag_markup)
+{
+    assert(!body->compiled);
+    vm_assembler_write_tag_markup(body->as.intermediate.code, tag_markup);
+    parse_context_set_parent_tag(parse_context, tag_markup);
+}
+
+static VALUE internal_block_body_parse(block_body_t *body, parse_context_t *parse_context)
 {
     tokenizer_t *tokenizer = parse_context->tokenizer;
     token_t token;
-    tag_markup_t unknown_tag = { Qnil, Qnil };
+    VALUE unknown_tag = Qnil;
     int render_score_increment = 0;
 
     while (true) {
@@ -209,7 +223,7 @@ static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_
 
                 if (name_len == 0) {
                     VALUE str = rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding);
-                    unknown_tag = (tag_markup_t) { str, str };
+                    unknown_tag = tag_markup_new(token_start_line_number, str, str, true);
                     goto loop_break;
                 }
 
@@ -224,8 +238,9 @@ static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_
                     tokenizer_setup_for_liquid_tag(tokenizer, markup_start, end, line_number);
                     unknown_tag = internal_block_body_parse(body, parse_context);
                     *tokenizer = saved_tokenizer;
-                    if (unknown_tag.name != Qnil) {
-                        rb_funcall(cLiquidBlockBody, intern_unknown_tag_in_liquid_tag, 2, unknown_tag.name, parse_context->ruby_obj);
+                    if (RTEST(unknown_tag)) {
+                        rb_funcall(cLiquidBlockBody, intern_unknown_tag_in_liquid_tag, 2,
+                                   tag_markup_get_tag_name(unknown_tag), parse_context->ruby_obj);
                         goto loop_break;
                     }
                     break;
@@ -238,13 +253,18 @@ static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_
                 VALUE markup = rb_enc_str_new(markup_start, end - markup_start, utf8_encoding);
 
                 if (tag_class == Qnil) {
-                    unknown_tag = (tag_markup_t) { tag_name, markup };
+                    unknown_tag = tag_markup_new(token_start_line_number, tag_name, markup, true);
                     goto loop_break;
                 }
 
+                VALUE tag_markup = tag_markup_new(token_start_line_number, tag_name, markup, false);
+                block_body_push_tag_markup(body, parse_context->ruby_obj, tag_markup);
+
                 VALUE new_tag = rb_funcall(tag_class, intern_parse, 4,
                         tag_name, markup, parse_context->tokenizer_obj, parse_context->ruby_obj);
 
+                parse_context_set_parent_tag(parse_context->ruby_obj, parse_context->parent_tag);
+
                 if (body->as.intermediate.blank && !RTEST(rb_funcall(new_tag, intern_is_blank, 0)))
                     body->as.intermediate.blank = false;
 
@@ -256,7 +276,7 @@ static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_
                     tokenizer->raw_tag_body = NULL;
                     tokenizer->raw_tag_body_len = 0;
                 } else {
-                    vm_assembler_add_write_node(body->as.intermediate.code, new_tag);
+                    block_body_add_node(body, new_tag);
                 }
 
                 render_score_increment += 1;
@@ -290,6 +310,7 @@ static void ensure_intermediate_not_parsing(block_body_t *body)
 static VALUE block_body_parse(VALUE self, VALUE tokenizer_obj, VALUE parse_context_obj)
 {
     parse_context_t parse_context = {
+        .parent_tag = parse_context_get_parent_tag(parse_context_obj),
         .tokenizer_obj = tokenizer_obj,
         .ruby_obj = parse_context_obj,
     };
@@ -303,10 +324,25 @@ static VALUE block_body_parse(VALUE self, VALUE tokenizer_obj, VALUE parse_conte
     }
     vm_assembler_remove_leave(body->as.intermediate.code); // to extend block
 
-    tag_markup_t unknown_tag = internal_block_body_parse(body, &parse_context);
+    VALUE unknown_tag = internal_block_body_parse(body, &parse_context);
     vm_assembler_add_leave(body->as.intermediate.code);
 
-    return rb_yield_values(2, unknown_tag.name, unknown_tag.markup);
+    VALUE tag_name = Qnil;
+    VALUE markup = Qnil;
+    if (RTEST(unknown_tag)) {
+        tag_name = tag_markup_get_tag_name(unknown_tag);
+        markup = tag_markup_get_markup(unknown_tag);
+        block_body_push_tag_markup(body, parse_context_obj, unknown_tag);
+    }
+
+    VALUE block_ret = rb_yield_values(2, tag_name, markup);
+
+    if (RTEST(parse_context.parent_tag) && !body->as.intermediate.bound_to_tag) {
+        body->as.intermediate.bound_to_tag = true;
+        tag_markup_set_block_body(parse_context.parent_tag, self, body);
+    }
+
+    return block_ret;
 }
 
 
@@ -352,7 +388,8 @@ static VALUE block_body_render_to_output_buffer(VALUE self, VALUE context, VALUE
     ensure_body_compiled(body);
     document_body_entry_t *entry = &body->as.compiled.document_body_entry;
 
-    liquid_vm_render(document_body_get_block_body_header_ptr(entry), document_body_get_constants_ptr(entry), context, output);
+    liquid_vm_render(document_body_get_block_body_header_ptr(entry), document_body_get_constants_ptr(entry),
+                     (const VALUE *)body->tags.data, context, output);
     return output;
 }
 
@@ -379,6 +416,7 @@ static VALUE block_body_remove_blank_strings(VALUE self)
         rb_raise(rb_eRuntimeError, "remove_blank_strings only support being called on a blank block body");
     }
 
+    VALUE *tags_ptr = (VALUE *)body->tags.data;
     VALUE *const_ptr = (VALUE *)body->as.intermediate.code->constants.data;
     uint8_t *ip = body->as.intermediate.code->instructions.data;
 
@@ -394,7 +432,7 @@ static VALUE block_body_remove_blank_strings(VALUE self)
                 body->as.intermediate.render_score--;
             }
         }
-        liquid_vm_next_instruction((const uint8_t **)&ip, (const VALUE **)&const_ptr);
+        liquid_vm_next_instruction((const uint8_t **)&ip, (const VALUE **)&const_ptr, (const VALUE **)&tags_ptr);
     }
 
     return Qnil;
@@ -425,6 +463,7 @@ static VALUE block_body_nodelist(VALUE self)
     VALUE nodelist = rb_ary_new_capa(body_header->render_score);
 
     const VALUE *const_ptr = document_body_get_constants_ptr(entry);
+    const VALUE *tags_ptr = (VALUE *)body->tags.data;
     const uint8_t *ip = block_body_instructions_ptr(body_header);
     while (true) {
         switch (*ip) {
@@ -448,15 +487,15 @@ static VALUE block_body_nodelist(VALUE self)
             }
             case OP_WRITE_NODE:
             {
-                rb_ary_push(nodelist, const_ptr[0]);
+                rb_ary_push(nodelist, tags_ptr[0]);
                 break;
             }
 
             case OP_RENDER_VARIABLE_RESCUE:
                 rb_ary_push(nodelist, variable_placeholder);
                 break;
         }
-        liquid_vm_next_instruction(&ip, &const_ptr);
+        liquid_vm_next_instruction(&ip, &const_ptr, &tags_ptr);
     }
 loop_break:
 
@@ -473,7 +512,7 @@ static VALUE block_body_disassemble(VALUE self)
     block_body_header_t *header = document_body_get_block_body_header_ptr(entry);
     const uint8_t *start_ip = block_body_instructions_ptr(header);
     return vm_assembler_disassemble(start_ip, start_ip + header->instructions_bytes,
-                                    document_body_get_constants_ptr(entry));
+                                    document_body_get_constants_ptr(entry), (const VALUE *)body->tags.data);
 }
 
 

ext/liquid_c/block.h

@@ -7,6 +7,7 @@
 typedef struct block_body {
     bool compiled;
     VALUE obj;
+    c_buffer_t tags;
 
     union {
         struct {
@@ -18,6 +19,7 @@ typedef struct block_body {
             vm_assembler_pool_t *vm_assembler_pool;
             bool blank;
             bool root;
+            bool bound_to_tag;
             unsigned int render_score;
             vm_assembler_t *code;
         } intermediate;
@@ -26,9 +28,9 @@ typedef struct block_body {
 
 void liquid_define_block_body();
 
-static inline uint8_t *block_body_instructions_ptr(block_body_header_t *body)
+static inline uint8_t *block_body_instructions_ptr(block_body_header_t *body_header)
 {
-    return ((uint8_t *)body) + body->instructions_offset;
+    return (uint8_t *)&body_header[1];
 }
 
 #endif

ext/liquid_c/c_buffer.c

@@ -16,14 +16,16 @@ static void c_buffer_expand_for_write(c_buffer_t *buffer, size_t write_size)
     buffer->capacity_end = buffer->data + capacity;
 }
 
-void c_buffer_zero_pad_for_alignment(c_buffer_t *buffer, size_t alignment)
+size_t c_buffer_zero_pad_for_alignment(c_buffer_t *buffer, size_t alignment)
 {
     size_t unaligned_bytes = c_buffer_size(buffer) % alignment;
     if (unaligned_bytes) {
         size_t pad_size = alignment - unaligned_bytes;
         uint8_t *padding = c_buffer_extend_for_write(buffer, pad_size);
         memset(padding, 0, pad_size);
+        return pad_size;
     }
+    return 0;
 }
 
 void c_buffer_reserve_for_write(c_buffer_t *buffer, size_t write_size)

ext/liquid_c/c_buffer.h

@@ -40,7 +40,7 @@ static inline size_t c_buffer_capacity(const c_buffer_t *buffer)
     return buffer->capacity_end - buffer->data;
 }
 
-void c_buffer_zero_pad_for_alignment(c_buffer_t *buffer, size_t alignment);
+size_t c_buffer_zero_pad_for_alignment(c_buffer_t *buffer, size_t alignment);
 
 void c_buffer_reserve_for_write(c_buffer_t *buffer, size_t write_size);
 void c_buffer_write(c_buffer_t *buffer, void *data, size_t size);

ext/liquid_c/document_body.c

@@ -3,6 +3,7 @@
 #include "liquid.h"
 #include "vm_assembler.h"
 #include "document_body.h"
+#include "tag_markup.h"
 
 static VALUE cLiquidCDocumentBody;
 
@@ -50,6 +51,41 @@ VALUE document_body_new_instance()
     return rb_class_new_instance(0, NULL, cLiquidCDocumentBody);
 }
 
+static void document_body_write_tag_markup(document_body_t *body, VALUE tag_markup_obj, bool last)
+{
+    tag_markup_t *tag_markup;
+    TagMarkup_Get_Struct(tag_markup_obj, tag_markup);
+
+    uint32_t tag_name_len = (uint32_t)RSTRING_LEN(tag_markup->tag_name);
+    uint32_t markup_len = (uint32_t)RSTRING_LEN(tag_markup->markup);
+    uint32_t total_len = sizeof(tag_markup_header_t) + tag_name_len + markup_len;
+    assert(c_buffer_size(&body->buffer) % alignof(tag_markup_header_t) == 0);
+    tag_markup_header_t *header = c_buffer_extend_for_write(&body->buffer, total_len);
+    if (!last) {
+        total_len += (uint32_t)c_buffer_zero_pad_for_alignment(&body->buffer, alignof(tag_markup_header_t));
+    }
+    char *name = (char *)&header[1];
+
+    header->flags = tag_markup->flags;
+    header->line_number = tag_markup->line_number;
+    header->tag_name_len = tag_name_len;
+    header->markup_len = markup_len;
+    header->next_tag_offset = last ? 0 : total_len;
+    if (tag_markup->block_body) {
+        if (!tag_markup->block_body->compiled) {
+            rb_raise(rb_eRuntimeError, "child %"PRIsVALUE" has not been frozen before the parent", tag_markup->block_body_obj);
+        }
+
+        header->block_body_offset = (uint32_t)tag_markup->block_body->as.compiled.document_body_entry.buffer_offset;
+    } else {
+        header->block_body_offset = BUFFER_OFFSET_UNDEF;
+    }
+
+    memcpy(name, RSTRING_PTR(tag_markup->tag_name), tag_name_len);
+    char *markup = name + tag_name_len;
+    memcpy(markup, RSTRING_PTR(tag_markup->markup), markup_len);
+}
+
 void document_body_write_block_body(VALUE self, bool blank, uint32_t render_score, vm_assembler_t *code, document_body_entry_t *entry)
 {
     document_body_t *body;
@@ -60,22 +96,40 @@ void document_body_write_block_body(VALUE self, bool blank, uint32_t render_scor
     entry->body = body;
     entry->buffer_offset = c_buffer_size(&body->buffer);
 
-    assert(c_buffer_size(&code->constants) % sizeof(VALUE) == 0);
-    uint32_t constants_len = (uint32_t)(c_buffer_size(&code->constants) / sizeof(VALUE));
+    size_t instructions_byte_size = c_buffer_size(&code->instructions);
+    size_t header_and_instructions_size = sizeof(block_body_header_t) + instructions_byte_size;
+    block_body_header_t *buf_block_body = c_buffer_extend_for_write(&body->buffer, header_and_instructions_size);
+    uint8_t *instructions = (uint8_t *)&buf_block_body[1];
 
-    block_body_header_t *buf_block_body = c_buffer_extend_for_write(&body->buffer, sizeof(block_body_header_t));
-    buf_block_body->instructions_offset = (uint32_t)sizeof(block_body_header_t);
-    buf_block_body->instructions_bytes = (uint32_t)c_buffer_size(&code->instructions);
-    buf_block_body->constants_offset = (uint32_t)RARRAY_LEN(body->constants);
-    buf_block_body->constants_len = constants_len;
     buf_block_body->flags = 0;
     if (blank) buf_block_body->flags |= BLOCK_BODY_HEADER_FLAG_BLANK;
     buf_block_body->render_score = render_score;
     buf_block_body->max_stack_size = code->max_stack_size;
+    buf_block_body->instructions_bytes = (uint32_t)instructions_byte_size;
 
-    c_buffer_concat(&body->buffer, &code->instructions);
+    assert(c_buffer_size(&code->constants) % sizeof(VALUE) == 0);
+    uint32_t constants_len = (uint32_t)(c_buffer_size(&code->constants) / sizeof(VALUE));
+    buf_block_body->constants_offset = (uint32_t)RARRAY_LEN(body->constants);
+    buf_block_body->constants_len = constants_len;
 
     rb_ary_cat(body->constants, (VALUE *)code->constants.data, constants_len);
+
+    memcpy(instructions, code->instructions.data, instructions_byte_size);
+
+    assert(c_buffer_size(&code->tag_markups) % sizeof(VALUE) == 0);
+    uint32_t tags_len = (uint32_t)(c_buffer_size(&code->tag_markups) / sizeof(VALUE));
+    if (tags_len > 0) {
+        buf_block_body->first_tag_offset = (uint32_t)header_and_instructions_size;
+        buf_block_body->first_tag_offset += (uint32_t)c_buffer_zero_pad_for_alignment(&body->buffer, alignof(tag_markup_header_t));
+
+        uint32_t i;
+        for (i = 0; i < tags_len - 1; i++) {
+            document_body_write_tag_markup(body, ((VALUE *)code->tag_markups.data)[i], false);
+        }
+        document_body_write_tag_markup(body, ((VALUE *)code->tag_markups.data)[i], true);
+    } else {
+        buf_block_body->first_tag_offset = 0;
+    }
 }
 
 void liquid_define_document_body()

ext/liquid_c/document_body.h

@@ -5,8 +5,8 @@
 #include "vm_assembler.h"
 
 typedef struct block_body_header {
-    uint32_t instructions_offset;
     uint32_t instructions_bytes;
+    uint32_t first_tag_offset;
     uint32_t constants_offset;
     uint32_t constants_len;
     uint32_t flags;
@@ -17,6 +17,9 @@ typedef struct block_body_header {
 #define BLOCK_BODY_HEADER_FLAG_BLANK (1 << 0)
 #define BLOCK_BODY_HEADER_BLANK_P(header) (header->flags & BLOCK_BODY_HEADER_FLAG_BLANK)
 
+#define BUFFER_OFFSET_UNDEF UINT32_MAX
+#define BUFFER_OFFSET_UNDEF_P(val) (val == BUFFER_OFFSET_UNDEF)
+
 typedef struct document_body {
     VALUE self;
     VALUE constants;