Solved: Migrated 41 sites, and now apache won't start

I migrated a whole bunch of sites from PLESK to ISPCONFIG, but now apache won't start.
After migration actually it was working fine, since the migration tool doesn't migrate paid SSL certificates, I was doing that manually, copy from the certificate, paste into site configuration, selecting Save Certificate.
After a while, the web server died. Nothing much in the logs, even with log mode set to debug httpd.conf.
It just says configuration failed or something.
I'm now running it with strace, but it's taking forever.
I can't remember which ssl I last entered, but I wrote a small program and it found problems with THIRTY SIX certificates!
I can't understand how so many of them could go corrupt on me!
here is the quick hack I wrote.. in rexx because I'm a mainframe guy.

#!/bin/rexx
'rxqueue /clear'
cf=''
ck=''
'grep SSLCertificate /etc/httpd/conf/sites-available/*|rxqueue'
do while queued()>0
parse pull fn cmd rest
if cmd='SSLCertificateFile' then cf = rest
if cmd='SSLCertificateKeyFile' then ck = rest
if cf>'' & ck>'' then do
'openssl x509 -in' cf '-text -noout'
rq = rc
'openssl rsa -in' ck ' -check -noout'
if rc<>0 | rq<>0 then rr=lineout('err',fn)
cf=''
ck=''
end
end

So I got 36 sites (!) in the 'err' file. I can't understand and much less how to fix this and get apache running again..

 

Here is the end of the strace.. The 'canimkendim' site seems to be the culprit, but surprisingly it passes the openssl checks above..

0
read(252, "", 4096) = 0
close(252) = 0
openat(AT_FDCWD, "/var/www/clients/client41/web28/ssl/drakciger.com-le.key", O_RDONLY) = 252
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
fstat(252, {st_mode=S_IFREG|0750, st_size=241, ...}) = 0
lseek(252, 0, SEEK_SET) = 0
read(252, "-----BEGIN PRIVATE KEY-----\nMIGH"..., 4096) = 241
close(252) = 0
getpid() = 7640
gettid() = 7640
write(181, "[Sat Mar 08 22:08:28.084162 2025"..., 341) = 341
getpid() = 7640
gettid() = 7640
write(181, "[Sat Mar 08 22:08:28.084210 2025"..., 256) = 256
openat(AT_FDCWD, "/var/www/clients/client41/web28/ssl/drakciger.com-le.crt", O_RDONLY) = 252
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
fstat(252, {st_mode=S_IFREG|0750, st_size=2860, ...}) = 0
lseek(252, 0, SEEK_SET) = 0
read(252, "-----BEGIN CERTIFICATE-----\nMIID"..., 4096) = 2860
lseek(252, 2860, SEEK_SET) = 2860
lseek(252, 0, SEEK_SET) = 0
read(252, "-----BEGIN CERTIFICATE-----\nMIID"..., 4096) = 2860
read(252, "", 4096) = 0
lseek(252, 0, SEEK_SET) = 0
read(252, "-----BEGIN CERTIFICATE-----\nMIID"..., 4096) = 2860
read(252, "", 4096) = 0
close(252) = 0
openat(AT_FDCWD, "/var/www/clients/client41/web28/ssl/drakciger.com-le.crt", O_RDONLY) = 252
fstat(252, {st_mode=S_IFREG|0750, st_size=2860, ...}) = 0
read(252, "-----BEGIN CERTIFICATE-----\nMIID"..., 4096) = 2860
read(252, "", 4096) = 0
close(252) = 0
getpid() = 7640
gettid() = 7640
write(201, "[Sat Mar 08 22:08:28.084656 2025"..., 130) = 130
openat(AT_FDCWD, "/etc/crypto-policies/back-ends/openssl.config", O_RDONLY) = 252
fstat(252, {st_mode=S_IFREG|0644, st_size=151, ...}) = 0
read(252, "@SECLEVEL=2:kEECDH:kRSA:kEDH:kPS"..., 4096) = 151
close(252) = 0
getpid() = 7640
getpid() = 7640
getpid() = 7640
getpid() = 7640
getpid() = 7640
gettid() = 7640
write(201, "[Sat Mar 08 22:08:28.085087 2025"..., 134) = 134
getpid() = 7640
gettid() = 7640
write(201, "[Sat Mar 08 22:08:28.085131 2025"..., 127) = 127
openat(AT_FDCWD, "/var/www/clients/client77/web219/ssl/canimkendim.org.crt", O_RDONLY) = 252
fstat(252, {st_mode=S_IFREG|0644, st_size=3965, ...}) = 0
read(252, "-----BEGIN CERTIFICATE-----\r\nMII"..., 4096) = 3965
read(252, "", 4096) = 0
close(252) = 0
openat(AT_FDCWD, "/var/www/clients/client77/web219/ssl/canimkendim.org.key", O_RDONLY) = 252
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
lseek(252, 0, SEEK_CUR) = 0
fstat(252, {st_mode=S_IFREG|0400, st_size=1730, ...}) = 0
lseek(252, 0, SEEK_SET) = 0
read(252, "-----BEGIN PRIVATE KEY-----\r\nMII"..., 4096) = 1730
read(252, "", 4096) = 0
close(252) = 0
getpid() = 7640
gettid() = 7640
write(201, "[Sat Mar 08 22:08:28.085605 2025"..., 261) = 261
write(2, "AH00016: Configuration Failed\n", 30) = 30
pselect6(0, NULL, NULL, NULL, {tv_sec=0, tv_nsec=10000000}, NULL) = 0 (Timeout)
close(251) = 0
close(250) = 0
close(249) = 0
kill(9855, SIGTERM) = 0
close(247) = 0

 

guess what fixed it! The last file the strace accessed, the damn /usr/share/crypto-policies/DEFAULT/openssl.txt file! (httpd is accessing a symlink /etc/crypto-policies/back-ends/openssl.config) I got one from the old server and overwrote it, and httpd started up!
Insane!

 

I also found one invalid certificate that's causing the crash. when I enable that certificate, apache won't start. so strange.