I'm currently trying to write a C 3D software rendering engine from scratch just for fun and to have an insight on what OpenGL does behind the scene and what 90's programmers had to do on DOS.

I have written my own matrix library and tested it without noticing any issues, but when I tried projecting the vertices of a simple 2x2 cube at 0,0 as seen by a basic camera at 0,0,10, the cube seems to appear way bigger than the application's window. If I scale the vertices' coordinates down by 8 times I can see a proper cube centered on the screen. This cube doesn't seem to be in perspective: wheen seen from the front, the back vertices pe rfectly overlap with the front ones, so I'm quite sure it's not correct.

this is how I create the view and projection matrices (vec4_initd initializes the vectors with w=0, vec4_initw initializes the vectors with w=1):

void mat4_lookatlh(mat4 *m, const vec4 *pos, const vec4 *target, const vec4 *updirection) {
    vec4 fwd, right, up;

    // fwd = norm(pos - target)
    fwd = *target;
    vec4_sub(&fwd, pos);
    vec4_norm(&fwd);

    // right = norm(cross(updirection, fwd))
    vec4_cross(updirection, &fwd, &right);
    vec4_norm(&right);

    // up = cross(right, forward)
    vec4_cross(&fwd, &right, &up);

    // orientation and translation matrices combined
    vec4_initd(&m->a, right.x, up.x, fwd.x);
    vec4_initd(&m->b, right.y, up.y, fwd.y);
    vec4_initd(&m->c, right.z, up.z, fwd.z);
    vec4_initw(&m->d, -vec4_dot(&right, pos), -vec4_dot(&up, pos), -vec4_dot(&fwd, pos));
}

void mat4_perspectivefovrh(mat4 *m, float fovdegrees, float aspectratio, float near, float far) {
    float h = 1.f / tanf(ftoradians(fovdegrees / 2.f));
    float w = h / aspectratio;

    vec4_initd(&m->a, w, 0.f, 0.f);
    vec4_initd(&m->b, 0.f, h, 0.f);
    vec4_initw(&m->c, 0.f, 0.f, -far / (near - far));
    vec4_initd(&m->d, 0.f, 0.f, (near * far) / (near - far));
}

this is how I project my vertices:

void device_project(device *d, const vec4 *coord, const mat4 *transform, int *projx, int *projy) {
    vec4 result;

    mat4_mul(transform, coord, &result);
    *projx = result.x * d->w + d->w / 2;
    *projy = result.y * d->h + d->h / 2;
}

void device_rendervertices(device *d, const camera *camera, const mesh meshes[], int nmeshes, const rgba *color) {
    int i, j;
    mat4 view, projection, world, transform, projview;
    mat4 translation, rotx, roty, rotz, transrotz, transrotzy;
    int projx, projy;

    // vec4_unity = (0.f, 1.f, 0.f, 0.f)
    mat4_lookatlh(&view, &camera->pos, &camera->target, &vec4_unity);
    mat4_perspectivefovrh(&projection, 45.f, (float)d->w / (float)d->h, 0.1f, 1.f);

    for (i = 0; i < nmeshes; i++) {
        // world matrix = translation * rotz * roty * rotx
        mat4_translatev(&translation, meshes[i].pos);
        mat4_rotatex(&rotx, ftoradians(meshes[i].rotx));
        mat4_rotatey(&roty, ftoradians(meshes[i].roty));
        mat4_rotatez(&rotz, ftoradians(meshes[i].rotz));
        mat4_mulm(&translation, &rotz, &transrotz); // transrotz = translation * rotz
        mat4_mulm(&transrotz, &roty, &transrotzy); // transrotzy = transrotz * roty = translation * rotz * roty
        mat4_mulm(&transrotzy, &rotx, &world); // world = transrotzy * rotx = translation * rotz * roty * rotx

        // transform matrix
        mat4_mulm(&projection, &view, &projview); // projview = projection * view
        mat4_mulm(&projview, &world, &transform); // transform = projview * world = projection * view * world

        for (j = 0; j < meshes[i].nvertices; j++) {
            device_project(d, &meshes[i].vertices[j], &transform, &projx, &projy);
            device_putpixel(d, projx, projy, color);
        }
    }
}

this is how the cube and camera are initialized:

// test mesh
cube = &meshlist[0];
mesh_init(cube, "Cube", 8);
cube->rotx = 0.f;
cube->roty = 0.f;
cube->rotz = 0.f;
vec4_initw(&cube->pos, 0.f, 0.f, 0.f);
vec4_initw(&cube->vertices[0], -1.f, 1.f, 1.f);
vec4_initw(&cube->vertices[1], 1.f, 1.f, 1.f);
vec4_initw(&cube->vertices[2], -1.f, -1.f, 1.f);
vec4_initw(&cube->vertices[3], -1.f, -1.f, -1.f);
vec4_initw(&cube->vertices[4], -1.f, 1.f, -1.f);
vec4_initw(&cube->vertices[5], 1.f, 1.f, -1.f);
vec4_initw(&cube->vertices[6], 1.f, -1.f, 1.f);
vec4_initw(&cube->vertices[7], 1.f, -1.f, -1.f);

// main camera
vec4_initw(&maincamera.pos, 0.f, 0.f, 10.f);
maincamera.target = vec4_zerow;

and, just to be sure, this is how I compute matrix multiplications:

void mat4_mul(const mat4 *m, const vec4 *va, vec4 *vb) {
    vb->x = m->a.x * va->x + m->b.x * va->y + m->c.x * va->z + m->d.x * va->w;
    vb->y = m->a.y * va->x + m->b.y * va->y + m->c.y * va->z + m->d.y * va->w;
    vb->z = m->a.z * va->x + m->b.z * va->y + m->c.z * va->z + m->d.z * va->w;
    vb->w = m->a.w * va->x + m->b.w * va->y + m->c.w * va->z + m->d.w * va->w;
}

void mat4_mulm(const mat4 *ma, const mat4 *mb, mat4 *mc) {
     mat4_mul(ma, &mb->a, &mc->a);
     mat4_mul(ma, &mb->b, &mc->b);
     mat4_mul(ma, &mb->c, &mc->c);
     mat4_mul(ma, &mb->d, &mc->d);
}