Just noodling on the future Node.js and http

framework.js

'use strict'

// Notice no certs, one thing I have thought
// for a long time is that frameworks should
// directly have support for spinning up with
// a cert if none was provided.
require('h3xt')()
  .get('/', (req) => {
    // Access the associated session
    // req.session

    // No need to check if can push, frameworks does this
    // The framework has something like a ROOT so you can
    // resolve static files, but the content type negotiation
    // is something I think belongs in the underlying core api
    req.pushFile('favicon.ico')
    req.pushJSON('/foo.json', {
      message: await req.body()
    })

    // Frameworks could do whatever templating they saw fit,
    // delivering the resulting string/buffer to `req.respond()`
    // In this example I am assuming a "return response" approach,
    // and the `.sendFile` call would return a `Response`
    // object, and the middleware/routing layer would use that
    // object to actually send
    return req.sendFile('index.html', {
      // template data
    })
  })
  .listen(8443)

http-next.js

'use strict'
const http = require('http-next')
const fs = require('fs')

http.createServer({
  allowHTTP1: true,
  allowHTTP2: true,
  allowHTTP3: true,
  key: fs.readFileSync('localhost-privkey.pem'),
  cert: fs.readFileSync('localhost-cert.pem')
})
  .on('session', (session) => {
    // I am not even sure what an end user would do with session.
    // Is it ok to store application state on? Like if a user cookie
    // was present in a request, could you load the user metadata onto
    // the session object?
    // Seems like a very useful feature with session pinning on your LB
    // and something like session.locals = Object.create(null) where users
    // could load on to.
    // And none of this would apply in http1, so there is also that to consider.

    session.on('stream', (stream, headers, flags) => {
      // Do a push stream if allowed
      // This covers the cases for no support in http1, http2 settings
      // and could even go from true to false when http3's max push is hit
      if (stream.pushAllowed) {
        stream.pushStream({
          path: '/favicon.ico'
        }, (err, pushStream, headers) => {
          if (err) {
            throw err
          }

          // Even in HTTP1 noone liked the way statusCode works!
          // I honeslty think we can do it all in one method signature
          // again with this, it is like 90% uf use cases
          pushStream.respond(200, {
            'content-type': 'image/x-icon'
          }, ourPreloadedFaviconBuffer)
        })
      }

      // A conviencence method for consuming the entire body
      // This covers like 90% of use cases
      // Also, we could add .consumeAsJSON to map to the new .respondWithJSON?
      stream.consume((body) => {
        // Always check again, because if we hit the max streams this would change
        // Actually, I wonder if it would be better for `stream.pushStream` to do
        // this check internally then just return false if it was not allowed?
        if (stream.pushAllowed) {
          stream.pushStream({
            path: '/foo.json'
          }, (err, pushStream, headers) => {
            if (err) {
              throw err
            }

            // Again with this, it is like 90% uf use cases
            pushStream.respondWithJSON({
              message: body
            })
          })
        }

        // The basic response with html
        stream.respondWithFile('index.html', {
          // This should be infered from the file name if not specified
          // 'content-type': 'text/html; charset=utf-8'
        })
      })
    })
  })
  .listen(8443)

simple-framework-src.js

const http = require('http-next')

module.exports = function (opts) {
  return new Application()
}

class Application {
  constructor () {
    this.stack = []
    for (const method of http.METHODS) {
      this[method] = this.use.bind(this, method)
    }
    this.server = http.createServer()
    this.server.onRequest(this.onRequest)
  }

  use (method, path, fnc) {
    this.stack.push({ path, method, fnc })
  }

  async onRequest (ctx) {
    // Accept and parse bodies
    if (ctx.method !== 'HEAD' && ctx.method !== 'GET') {
      // Read body
      const _body = []
      for await (const chunk of ctx.body) {
        _body.push(chunk)
      }
      let body = Buffer.from(..._body)
      if (ctx.headers['content-type'] === 'application/json') {
        body = JSON.parse(body.toString())
      }
      ctx.body = body
    }

    for (const layer of this.stack) {
      if (layer.method && ctx.method !== layer.method) {
        continue
      }
      if (layer.path && ctx.path !== layer.path) {
        continue
      }

      let statusCode = 200
      let headers = {}
      let body = null

      const ret = await layer.fnc(ctx)

      if (ret === undefined) {
        continue
      }

      if (Number.isFinite(ret)) {
        statusCode = ret
        let msg = ''
        switch (statusCode) {
          case 200:
            msg = 'Success'
            break
          case 404:
            msg = 'Not Found'
            break
          case 500:
            msg = 'Server Error'
            break
        }
        body = Buffer.from(msg)
        headers = {
          'content-type': 'text/plain',
          'content-length': body.length
        }
      } else if (typeof ret !== 'undefined' && typeof ret === 'object') {
        body = Buffer.from(JSON.stringify(ret))
        headers = {
          'content-type': 'application/json',
          'content-length': body.length
        }
      } else if (typeof ret === 'string') {
        body = Buffer.from(ret, 'utf8')
        headers = {
          'content-type': 'text/plain',
          'content-length': body.length
        }
      }

      return ctx.respondWith(statusCode, headers, body)
    }
  }

  listen (...args) {
    return this.server.listen(...args)
  }
}

anything which I can leverage from "core" means less code to maintain

As a node core developer. Anything that can be implemented in frameworks means less code to maintain. 😄

As you notice from my comments I'm much in favor of a small core. However, this does provide motivation for something between core and the frameworks, where e.g. WG modules could come into play.

I don't quite agree with you on the Duplex thing but I think it's easier to discuss over a call.

(also that is the top priority in the "next 10 years of node" values conversation.

Maintainer experience is also on the list.

To conclude, I think we are generally in agreement on the approach we need to take.

I think we are in agreement I'm just looking for a layered approach. With a low level API which your higher level API could/would be implemented on top of either in core, WG packages or frameworks.

I think we are in agreement I'm just looking for a layered approach

Same! I think that the net layer might be where some of the things you are bringing up might live, no? My understanding was that the lowest one was what he was working on now, and that we would have a layer in between which had the http "primatives" and then we would build an api on top of that. Maybe I miss-understand what others thought of that diagram.

If the "high level/user api" box is where the http parser and everything from the current http1/2 api's lives I don't know how much that gets us. I think we need a better split there.

I think this does make it clear we need to have a firm understanding of that those lower two boxes in James diagram mean. Which is one of the reasons I wanted to know how to best contact you, because we are planning a video chat next week for him to present the work and to discuss.

TBQH, maybe we do need one more layer in there?

TBQH, maybe we do need one more layer in there?

Yea. That's what I'm thinking.

This is my thoughts:

transport |    protocol  |            core api         |        helper api        |    libraries/frameworks
-----------------------------------------------------------------------------------------------------------
net       | http1        |                             | low level framework      |
tls       | https/http2  | low level http generic API  | high level http generic  | high level framework
quic      | http3        |                             |                          |

Essentially 3-5 layers depending on what makes sense. At least conceptually. In practice it might be difficult, e.g. I'm not sure whether the transport and protocol layers should/can be separated in all cases.

I think what you are aiming for is the "helper api" layer while I'm looking for "core api". Whether it actually makes sense to separate these is worth discussing.

reasons I wanted to know how to best contact you

I hope you got the answer?

This is a great diagram! And totally aligns with how I have grown to think about it lately. One addition I would make is to add http1 and https to the quic protocol layer so we understand what the future will look like in a complete sense.

I might also think about naming a bit more, maybe: transport, protocol, foundation, user and framework apis? My nit pick on this is that the word core is ambiguous, as in the end I think we would expose all but the framework api as a part of "node core" which would lead the double usage of "core" to confuse people.

This is a great diagram! And totally aligns with how I have grown to think about it lately. One addition I would make is to add http1 and https to the quic protocol layer so we understand what the future will look like in a complete sense.

I might also think about naming a bit more, maybe: transport, protocol, foundation, user and framework apis? My nit pick on this is that the word core is ambiguous, as in the end I think we would expose all but the framework api as a part of "node core" which would lead the double usage of "core" to confuse people.

I agree with all of the above.

transport	protocol	core	working group	user
net	http	low level generic api	high level generic api	framework api
tls	https
quic	http2
	http3

Applying this to undici it would look something like:

transport	protocol	core	working group	user
net	http	dispatch	request, stream, connect, upgrade	got
tls	https

A few proposed updates from our call:

transport	protocol	core	working group	user
tcp	http	quic compat api	high level generic api	framework api
tls	https
quic

httpNext.createServer({})

http.createServer({
  protocol: 'quic+http1',
  port: 1234
}).on('session', (sess) => fmw.emit('session', sess))

http.createServer({
  protocol: 'https'
}).on('session', (sess) => fmw.emit('session', sess))