netty enclosed frame decoder

EnclosedFrameDecoder.java

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;

import java.util.List;

public class EnclosedFrameDecoder extends ByteToMessageDecoder {

    private final ByteBuf startFlag, endFlag;
    private final int startLength, endLength;

    public EnclosedFrameDecoder(byte[] startFlag, byte[] endFlag) {
        this(Unpooled.wrappedBuffer(startFlag), Unpooled.wrappedBuffer(endFlag));
    }

    public EnclosedFrameDecoder(ByteBuf startFlag, ByteBuf endFlag) {
        this.startFlag = startFlag;
        this.endFlag = endFlag;
        this.startLength = startFlag.readableBytes();
        this.endLength = endFlag.readableBytes();
    }

    @Override
    protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {
        int start = ByteBufUtil.indexOf(startFlag, in);
        if (start == -1) {
            return;
        }
        start += startLength;
        in.skipBytes(start - in.readerIndex());
        int end = ByteBufUtil.indexOf(endFlag, in);
        if (end == -1) {
            return;
        }
        int len = end - start;
        if (len < 1) {
            return;
        }
        out.add(in.readRetainedSlice(len));
        in.skipBytes(endLength);
    }
}

EnclosedFrameDecoderTest.java

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.Unpooled;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;

public class EnclosedFrameDecoderTest {

    @Test
    public void normalFrame() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x7c, (byte) 0xba, 0x04, 0x70, (byte) 0xa3, (byte) 0xf1, 0x69, 0x00, 0x01
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x7c, (byte) 0xba, 0x04, 0x70, (byte) 0xa3, (byte) 0xf1, 0x69});
        Assertions.assertEquals(expected, buf);
    }

    @Test
    public void multipleNormalFrame() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01,
                0x00, 0x00, 0x05, 0x71, 0x00, 0x01
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        Assertions.assertEquals(expected, buf);
    }

    @Test
    public void multipleNormalFrameMultipleCall() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01,
                0x00, 0x00, 0x05, 0x71, 0x72, 0x00, 0x01,
                0x00, 0x00, 0x06, 0x61, 0x00, 0x01,
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        Assertions.assertEquals(3, output.size());

        ByteBuf buf = (ByteBuf) output.get(0);
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertEquals(expected, buf);

        buf = (ByteBuf) output.get(1);
        expected = Unpooled.wrappedBuffer(new byte[] {0x05, 0x71, 0x72});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertEquals(expected, buf);

        buf = (ByteBuf) output.get(2);
        expected = Unpooled.wrappedBuffer(new byte[] {0x06, 0x61});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertEquals(expected, buf);
    }

    @Test
    public void multipleBetweenJunkBytes() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01,
                0x0E,
                0x00, 0x00, 0x05, 0x71, 0x72, 0x00, 0x01,
                0x3F, 0x2F,
                0x00, 0x00, 0x06, 0x61, 0x00, 0x01,
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        Assertions.assertEquals(3, output.size());

        ByteBuf buf = (ByteBuf) output.get(0);
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertEquals(expected, buf);

        buf = (ByteBuf) output.get(1);
        expected = Unpooled.wrappedBuffer(new byte[] {0x05, 0x71, 0x72});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertEquals(expected, buf);

        buf = (ByteBuf) output.get(2);
        expected = Unpooled.wrappedBuffer(new byte[] {0x06, 0x61});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertEquals(expected, buf);
    }

    @Test
    public void leadingUnknownBytes() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                (byte) 0x95, 0x51,
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }

    @Test
    public void leadingPartialFrame() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                (byte) 0x95, 0x51, 0x00, 0x01,
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }

    @Test
    public void trailingUnknownBytes() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01, 0x15, 0x51
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }

    @Test
    public void trailingUnknownBytes2() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01, 0x15, 0x51
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }

    @Test
    public void trailingPartialFrame() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x04, 0x70, 0x00, 0x01, 0x00, 0x00, 0x15, 0x51
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x04, 0x70});
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }

    @Test
    public void emptyFrame() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x00}, new byte[]{0x00, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x00, 0x00, 0x01
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        Assertions.assertEquals(0, output.size());
    }

    @Test
    public void singleChar() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{'*'}, new byte[]{'#'});

        var input = Unpooled.wrappedBuffer(new byte[] {
                '*', 0x00, 0x01, 0x02, '#'
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        Assertions.assertEquals(1, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        System.out.println(ByteBufUtil.hexDump(buf));
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x00, 0x01, 0x02});
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }

    @Test
    public void threeChar() throws Exception {
        var decoder = new EnclosedFrameDecoder(new byte[]{0x00, 0x01, 0x02}, new byte[]{0x03, 0x02, 0x01});

        var input = Unpooled.wrappedBuffer(new byte[] {
                0x00, 0x01, 0x02, 0x0F, 0x0E, 0x0D, 0x03, 0x02, 0x01,
                0x00, 0x01, 0x02, 0x1F, 0x1E, 0x1D, 0x03, 0x02, 0x01,
        });
        var output = new ArrayList<>();

        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        decoder.decode(null, input, output);
        Assertions.assertEquals(2, output.size());
        ByteBuf buf = (ByteBuf) output.get(0);
        ByteBuf expected = Unpooled.wrappedBuffer(new byte[] {0x0F, 0x0E, 0x0D});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));

        buf = (ByteBuf) output.get(1);
        expected = Unpooled.wrappedBuffer(new byte[] {0x1F, 0x1E, 0x1D});
        System.out.println(ByteBufUtil.hexDump(buf));
        Assertions.assertTrue(ByteBufUtil.equals(expected, buf));
    }
}