Zoom in on a Point (Using Scale and Translate)

const zoomIntensity = 0.2;

const canvas = document.getElementById("canvas");
let context = canvas.getContext("2d");
const width = 600;
const height = 200;

let scale = 1;
let originx = 0;
let originy = 0;
let visibleWidth = width;
let visibleHeight = height;


function draw(){
    // Clear screen to white.
    context.fillStyle = "white";
    context.fillRect(originx, originy, width/scale, height/scale);
    // Draw the black square.
    context.fillStyle = "black";
    context.fillRect(50, 50, 100, 100);

    // Schedule the redraw for the next display refresh.
    window.requestAnimationFrame(draw);
}
// Begin the animation loop.
draw();

canvas.onwheel = function (event){
    event.preventDefault();
    // Get mouse offset.
    const mousex = event.clientX - canvas.offsetLeft;
    const mousey = event.clientY - canvas.offsetTop;
    // Normalize mouse wheel movement to +1 or -1 to avoid unusual jumps.
    const wheel = event.deltaY < 0 ? 1 : -1;

    // Compute zoom factor.
    const zoom = Math.exp(wheel * zoomIntensity);
    
    // Translate so the visible origin is at the context's origin.
    context.translate(originx, originy);
  
    // Compute the new visible origin. Originally the mouse is at a
    // distance mouse/scale from the corner, we want the point under
    // the mouse to remain in the same place after the zoom, but this
    // is at mouse/new_scale away from the corner. Therefore we need to
    // shift the origin (coordinates of the corner) to account for this.
    originx -= mousex/(scale*zoom) - mousex/scale;
    originy -= mousey/(scale*zoom) - mousey/scale;
    
    // Scale it (centered around the origin due to the trasnslate above).
    context.scale(zoom, zoom);
    // Offset the visible origin to it's proper position.
    context.translate(-originx, -originy);

    // Update scale and others.
    scale *= zoom;
    visibleWidth = width / scale;
    visibleHeight = height / scale;
}

<canvas id="canvas" width="600" height="200"></canvas>

Zoom in on a mousewheel point (using scale and translate)

You were close to it, however it's better to store the x,y and scale separately and calculate the transforms based on those values. It makes things alot easier + saves resources (no need to lookup the dom properties over and over),

I've put the code into a nice module:

function ScrollZoom(container,max_scale,factor){
    var target = container.children().first()
    var size = {w:target.width(),h:target.height()}
    var pos = {x:0,y:0}
    var zoom_target = {x:0,y:0}
    var zoom_point = {x:0,y:0}
    var scale = 1
    target.css('transform-origin','0 0')
    target.on("mousewheel DOMMouseScroll",scrolled)

    function scrolled(e){
        var offset = container.offset()
        zoom_point.x = e.pageX - offset.left
        zoom_point.y = e.pageY - offset.top

        e.preventDefault();
        var delta = e.delta || e.originalEvent.wheelDelta;
        if (delta === undefined) {
          //we are on firefox
          delta = e.originalEvent.detail;
        }
        delta = Math.max(-1,Math.min(1,delta)) // cap the delta to [-1,1] for cross browser consistency

        // determine the point on where the slide is zoomed in
        zoom_target.x = (zoom_point.x - pos.x)/scale
        zoom_target.y = (zoom_point.y - pos.y)/scale

        // apply zoom
        scale += delta*factor * scale
        scale = Math.max(1,Math.min(max_scale,scale))

        // calculate x and y based on zoom
        pos.x = -zoom_target.x * scale + zoom_point.x
        pos.y = -zoom_target.y * scale + zoom_point.y


        // Make sure the slide stays in its container area when zooming out
        if(pos.x>0)
            pos.x = 0
        if(pos.x+size.w*scale<size.w)
            pos.x = -size.w*(scale-1)
        if(pos.y>0)
            pos.y = 0
         if(pos.y+size.h*scale<size.h)
            pos.y = -size.h*(scale-1)

        update()
    }

    function update(){
        target.css('transform','translate('+(pos.x)+'px,'+(pos.y)+'px) scale('+scale+','+scale+')')
    }
}

Use it by calling

new ScrollZoom($('#container'),4,0.5)

The parameters are:

1. container: The wrapper of the element to be zoomed. The script will
   look for the first child of the container and apply the transforms
   to it.
2. max_scale: The maximum scale (4 = 400% zoom)
3. factor: The zoom-speed (1 = +100% zoom per mouse wheel tick)

JSFiddle here

How to zoom on a point with JavaScript?

Really incredible, I actually did it.

window.onload = () => {    const STEP = 0.99;    const MAX_SCALE = 5;    const MIN_SCALE = 0.01;
    const red = document.getElementById("red");    const yellow = red.parentNode;
    let scale = 1;
    const rect = red.getBoundingClientRect();    const originCenterX = rect.x + rect.width / 2;    const originCenterY = rect.y + rect.height / 2;
    yellow.onwheel = (event) => {        event.preventDefault();
        const factor = event.deltaY;
        // If current scale is equal to or greater than MAX_SCALE, but you're still zoom in it, then return;        // If current scale is equal to or smaller than MIN_SCALE, but you're still zoom out it, then return;        // Can not use Math.max and Math.min here, think about it.        if ((scale >= MAX_SCALE && factor < 0) || (scale <= MIN_SCALE && factor > 0)) return;                const scaleChanged = Math.pow(STEP, factor);        scale *= scaleChanged;
        const rect = red.getBoundingClientRect();        const currentCenterX = rect.x + rect.width / 2;        const currentCenterY = rect.y + rect.height / 2;
        const mousePosToCurrentCenterDistanceX = event.clientX - currentCenterX;        const mousePosToCurrentCenterDistanceY = event.clientY - currentCenterY;
        const newCenterX = currentCenterX + mousePosToCurrentCenterDistanceX * (1 - scaleChanged);        const newCenterY = currentCenterY + mousePosToCurrentCenterDistanceY * (1 - scaleChanged);
        // All we are doing above is: getting the target center, then calculate the offset from origin center.        const offsetX = newCenterX - originCenterX;        const offsetY = newCenterY - originCenterY;
        // !!! Both translate and scale are relative to the original position and scale, not to the current.        red.style.transform = 'translate(' + offsetX + 'px, ' + offsetY + 'px)' + 'scale(' + scale + ')';    }}

.yellow {  background-color: yellow;  width: 200px;  height: 100px;
  margin-left: 50px;  margin-top: 50px;
  position: absolute;}
.red {  background-color: red;  width: 200px;  height: 100px;
  position: absolute;}

<div class="yellow">    <div id="red" class="red"></div></div>

<!DOCTYPE html>
<html lang="en">

<head>
  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>

  <style>
    .container {
      background-color: lightgrey;
    }
    
    .stage {
      height: 90vh;
      width: 100%;
      overflow: hidden;
    }
    
    #image {
      transform-origin: 50% 50%;
      height: auto;
      width: 40%;
      cursor: grab;
    }
    
    .actions {
      display: flex;
      position: absolute;
      bottom: 0;
      left: 0;
      height: 1.5rem;
      width: 100%;
      background-color: lightgrey;
    }
    
    .action {
      margin-right: 1rem;
    }
  </style>
</head>

<body>
  <div class="container">
    <div class="stage">
      <img id="image" src="https://cdn.pixabay.com/photo/2018/01/14/23/12/nature-3082832__480.jpg" />
    </div>
    <div class="actions">
      <div class="action">
        <label for="rotate">Rotate </label>
        <input type="range" id="rotate" name="rotate" min="0" max="360">
        <button onclick="reset()">Reset All</button>
      </div>
    </div>
  </div>
  <script>
    const img = document.getElementById('image');
    const rotate = document.getElementById('rotate');
    let mouseX;
    let mouseY;
    let mouseTX;
    let mouseTY;
    let startXOffset = 222.6665;
    let startYOffset = 224.713;
    let startX = 0;
    let startY = 0;
    let panning = false;

    const ts = {
      scale: 1,
      rotate: 0,
      translate: {
        x: 0,
        y: 0
      }
    };

    rotate.oninput = function(event) {
      event.preventDefault();
      ts.rotate = event.target.value;
      setTransform();
    };

    img.onwheel = function(event) {
      event.preventDefault();
      //need more handling  to avoid fast scrolls
      var func = img.onwheel;
      img.onwheel = null;

      let rec = img.getBoundingClientRect();
      let x = (event.clientX - rec.x) / ts.scale;
      let y = (event.clientY - rec.y) / ts.scale;

      let delta = (event.wheelDelta ? event.wheelDelta : -event.deltaY);
      ts.scale = (delta > 0) ? (ts.scale + 0.2) : (ts.scale - 0.2);

      //let m = (ts.scale - 1) / 2;
      let m = (delta > 0) ? 0.1 : -0.1;
      ts.translate.x += (-x * m * 2) + (img.offsetWidth * m);
      ts.translate.y += (-y * m * 2) + (img.offsetHeight * m);

      setTransform();
      img.onwheel = func;
    };

    img.onmousedown = function(event) {
      event.preventDefault();
      panning = true;
      img.style.cursor = 'grabbing';
      mouseX = event.clientX;
      mouseY = event.clientY;
      mouseTX = ts.translate.x;
      mouseTY = ts.translate.y;
    };

    img.onmouseup = function(event) {
      panning = false;
      img.style.cursor = 'grab';
    };

    img.onmousemove = function(event) {
      event.preventDefault();
      let rec = img.getBoundingClientRect();
      let xx = event.clientX - rec.x;
      let xy = event.clientY - rec.y;

      const x = event.clientX;
      const y = event.clientY;
      pointX = (x - startX);
      pointY = (y - startY);
      if (!panning) {
        return;
      }
      ts.translate.x =
        mouseTX + (x - mouseX);
      ts.translate.y =
        mouseTY + (y - mouseY);
      setTransform();
    };

    function setTransform() {
      const steps = `translate(${ts.translate.x}px,${ts.translate.y}px) scale(${ts.scale}) rotate(${ts.rotate}deg) translate3d(0,0,0)`;
      //console.log(steps);
      img.style.transform = steps;
    }

    function reset() {
      ts.scale = 1;
      ts.rotate = 0;
      ts.translate = {
        x: 0,
        y: 0
      };
      rotate.value = 180;
      img.style.transform = 'none';
    }

    setTransform();
  </script>
</body>

</html>

var scale = 1,
    panning = false,
    xoff = 0,
    yoff = 0,
    start = {x: 0, y: 0},
    doc = document.getElementById("document");

function setTransform() {
  doc.style.transform = "translate(" + xoff + "px, " + yoff + "px) scale(" + scale + ")";
}

doc.onmousedown = function(e) {
  e.preventDefault();
  start = {x: e.clientX - xoff, y: e.clientY - yoff};    
  panning = true;
}

doc.onmouseup = function(e) {
  panning = false;
}

doc.onmousemove = function(e) {
  e.preventDefault();         
  if (!panning) {
    return;
  }
  xoff = (e.clientX - start.x);
  yoff = (e.clientY - start.y);
  setTransform();
}

doc.onwheel = function(e) {
    e.preventDefault();
    // take the scale into account with the offset
    var xs = (e.clientX - xoff) / scale,
        ys = (e.clientY - yoff) / scale,
        delta = (e.wheelDelta ? e.wheelDelta : -e.deltaY);

    // get scroll direction & set zoom level
    (delta > 0) ? (scale *= 1.2) : (scale /= 1.2);

    // reverse the offset amount with the new scale
    xoff = e.clientX - xs * scale;
    yoff = e.clientY - ys * scale;

    setTransform();          
}

html, body {
  margin: 0;
  padding: 0;
  width: 100%;
  height: 100%;
  overflow: hidden;
}

#document {
  width: 100%;
  height: 100%;
  transform-origin: 0px 0px;
  transform: scale(1) translate(0px, 0px);
}

<div id="document">
  <img style="width: 100%"
       src="https://i.imgur.com/fHyEMsl.jpg"
       crossOrigin="" />
</div>

var current = {x: 0, y: 0, zoom: 1},    con = document.getElementById('container');    window.onclick = function(e) {    var coef = e.shiftKey || e.ctrlKey ? 0.5 : 2,        oz = current.zoom,        nz = current.zoom * coef,        /// offset of container        ox = 20,        oy = 20,        /// mouse cords        mx = e.clientX - ox,        my = e.clientY - oy,        /// calculate click at current zoom        ix = (mx - current.x) / oz,        iy = (my - current.y) / oz,        /// calculate click at new zoom        nx = ix * nz,        ny = iy * nz,        /// move to the difference        /// make sure we take mouse pointer offset into account!        cx = mx - nx,        cy = my - ny    ;    // update current    current.zoom = nz;    current.x = cx;    current.y = cy;    /// make sure we translate before scale!    con.style.transform        = 'translate3D('+cx+'px, '+cy+'px,0) '        + 'scale('+nz+')'    ;};

#container {    position: absolute;    left: 20px;    top: 20px;    width: 100%;    height: 100%;    transform-origin: 0 0 0;    transition: transform 0.3s;    transition-timing-function: ease-in-out;    transform: translate3D(0,0,0) scale(1);}
#item {    position: absolute;}

<div id="container">    <div id="item">        <img src="http://fadili.users.greyc.fr/demos/WaveRestore/EMInpaint/parrot_original.png" />    </div></div>

NSPoint eventLocation = [event locationInWindow];
NSPoint center = [self.view convertPoint:eventLocation fromView:nil];

double transX = 1.0 - center.x / (frame.size.width / 2.0);
double transY = 1.0 - center.y / (frame.size.height / 2.0);
GLKMatrix4 trans = GLKMatrix4Translate(GLKMatrix4Identity, transX, transY, 0);
GLKMatrix4 transBack = GLKMatrix4Translate(GLKMatrix4Identity, -transX, -transY, 0);
GLKMatrix4 scaleMatrix = GLKMatrix4Scale(GLKMatrix4Identity, self.zoomValue, self.zoomValue, 1);
GLKMatrix4 model = GLKMatrix4Multiply( transBack, GLKMatrix4Multiply( GLKMatrix4Multiply( scaleMatrix, trans ), _baseScaleMatrix ) );

const ctx = canvas.getContext("2d");
canvas.width = 500;
canvas.height = 500;
const rand = (m = 255, M = m + (m = 0)) => (Math.random() * (M - m) + m) | 0;


const objects = [];
for (let i = 0; i < 100; i++) {
  objects.push({x: rand(canvas.width), y: rand(canvas.height),w: rand(40),h: rand(40), col: `rgb(${rand()},${rand()},${rand()})`});
}

requestAnimationFrame(drawCanvas); 

const view = (() => {
  const matrix = [1, 0, 0, 1, 0, 0]; // current view transform
  var m = matrix;             // alias 
  var scale = 1;              // current scale
  var ctx;                    // reference to the 2D context
  const pos = { x: 0, y: 0 }; // current position of origin
  var dirty = true;
  const API = {
    set context(_ctx) { ctx = _ctx; dirty = true },
    apply() {
      if (dirty) { this.update() }
      ctx.setTransform(m[0], m[1], m[2], m[3], m[4], m[5])
    },
    get scale() { return scale },
    get position() { return pos },
    isDirty() { return dirty },
    update() {
      dirty = false;
      m[3] = m[0] = scale;
      m[2] = m[1] = 0;
      m[4] = pos.x;
      m[5] = pos.y;
    },
    pan(amount) {
      if (dirty) { this.update() }
       pos.x += amount.x;
       pos.y += amount.y;
       dirty = true;
    },
    scaleAt(at, amount) { // at in screen coords
      if (dirty) { this.update() }
      scale *= amount;
      pos.x = at.x - (at.x - pos.x) * amount;
      pos.y = at.y - (at.y - pos.y) * amount;
      dirty = true;
    },
  };
  return API;
})();
view.context = ctx;
function drawCanvas() {
    if (view.isDirty()) { 
        ctx.setTransform(1, 0, 0, 1, 0, 0); 
        ctx.clearRect(0, 0, canvas.width, canvas.height);

        view.apply(); // set the 2D context transform to the view
        for (i = 0; i < objects.length; i++) {
            var obj = objects[i];
            ctx.fillStyle = obj.col;
            ctx.fillRect(obj.x, obj.y, obj.h, obj.h);
        }
    }
    requestAnimationFrame(drawCanvas);
}


canvas.addEventListener("mousemove", mouseEvent, {passive: true});
canvas.addEventListener("mousedown", mouseEvent, {passive: true});
canvas.addEventListener("mouseup", mouseEvent, {passive: true});
canvas.addEventListener("mouseout", mouseEvent, {passive: true});
canvas.addEventListener("wheel", mouseWheelEvent, {passive: false});
const mouse = {x: 0, y: 0, oldX: 0, oldY: 0, button: false};
function mouseEvent(event) {
    if (event.type === "mousedown") { mouse.button = true }
    if (event.type === "mouseup" || event.type === "mouseout") { mouse.button = false }
    mouse.oldX = mouse.x;
    mouse.oldY = mouse.y;
    mouse.x = event.offsetX;
    mouse.y = event.offsetY    
    if(mouse.button) { // pan
        view.pan({x: mouse.x - mouse.oldX, y: mouse.y - mouse.oldY});
    }
}
function mouseWheelEvent(event) {
    var x = event.offsetX;
    var y = event.offsetY;
    if (event.deltaY < 0) { view.scaleAt({x, y}, 1.1) }
    else { view.scaleAt({x, y}, 1 / 1.1) }
    event.preventDefault();
}

body {
  background: gainsboro;
  margin: 0;
}
canvas {
  background: white;
  box-shadow: 1px 1px 1px rgba(0, 0, 0, .2);
}

<canvas id="canvas"></canvas>